2024-04-26T04:41:17.140 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-26T04:41:17.141 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --name teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595 --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595/orig.config.yaml 2024-04-26T04:41:17.202 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T04:41:17.299 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7674595' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 8320 sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 10/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a targets: smithi152.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOf23yzaLcFuBatbPqDCU5czzKwmjuLZHWW0K08Qqu5BTKYNWPN2wWKcCS7EVgkqEjARquo8CANWGgOYSOayGq4= smithi195.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAxaCkGt5M7WGAinUpMquxJmF3Z1AuEWKJlhtoLD5RyKvoNAlXMA4YEb/WHnr7pCmxlqgeE/DXuOwEVv/Geec1I= tasks: - pexec: all: - sudo dnf install runc -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_02:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-26T04:41:17.299 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa; will attempt to use it 2024-04-26T04:41:17.300 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks 2024-04-26T04:41:17.300 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-26T04:41:17.302 INFO:teuthology.task.internal:Checking packages... 2024-04-26T04:41:17.325 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a' 2024-04-26T04:41:17.325 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:41:17.325 INFO:teuthology.packaging:ref: None 2024-04-26T04:41:17.325 INFO:teuthology.packaging:tag: None 2024-04-26T04:41:17.325 INFO:teuthology.packaging:branch: squid 2024-04-26T04:41:17.325 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:17.325 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-26T04:41:17.621 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2447.gb22e2ebd 2024-04-26T04:41:17.623 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-26T04:41:17.629 INFO:teuthology.task.internal:no buildpackages task found 2024-04-26T04:41:17.629 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-26T04:41:17.673 INFO:teuthology.task.internal:Saving configuration 2024-04-26T04:41:17.695 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-26T04:41:17.701 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T04:41:17.726 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:36:02.908114', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOf23yzaLcFuBatbPqDCU5czzKwmjuLZHWW0K08Qqu5BTKYNWPN2wWKcCS7EVgkqEjARquo8CANWGgOYSOayGq4='} 2024-04-26T04:41:17.748 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi195.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:36:02.910888', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAxaCkGt5M7WGAinUpMquxJmF3Z1AuEWKJlhtoLD5RyKvoNAlXMA4YEb/WHnr7pCmxlqgeE/DXuOwEVv/Geec1I='} 2024-04-26T04:41:17.748 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-26T04:41:17.755 INFO:teuthology.task.internal:roles: ubuntu@smithi152.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-26T04:41:17.755 INFO:teuthology.task.internal:roles: ubuntu@smithi195.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-26T04:41:17.755 INFO:teuthology.run_tasks:Running task console_log... 2024-04-26T04:41:17.829 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f252cbdf0d0>, signals=[15]) 2024-04-26T04:41:17.830 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-26T04:41:17.836 INFO:teuthology.task.internal:Opening connections... 2024-04-26T04:41:17.836 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi152.front.sepia.ceph.com 2024-04-26T04:41:17.838 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:41:17.913 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi195.front.sepia.ceph.com 2024-04-26T04:41:17.914 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:41:17.989 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-26T04:41:17.995 DEBUG:teuthology.orchestra.run.smithi152:> uname -m 2024-04-26T04:41:18.013 INFO:teuthology.orchestra.run.smithi152.stdout:x86_64 2024-04-26T04:41:18.013 DEBUG:teuthology.orchestra.run.smithi152:> cat /etc/os-release 2024-04-26T04:41:18.069 INFO:teuthology.orchestra.run.smithi152.stdout:NAME="CentOS Stream" 2024-04-26T04:41:18.069 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION="9" 2024-04-26T04:41:18.069 INFO:teuthology.orchestra.run.smithi152.stdout:ID="centos" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION_ID="9" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:ANSI_COLOR="0;31" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:41:18.070 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:41:18.071 INFO:teuthology.lock.ops:Updating smithi152.front.sepia.ceph.com on lock server 2024-04-26T04:41:18.099 DEBUG:teuthology.orchestra.run.smithi195:> uname -m 2024-04-26T04:41:18.117 INFO:teuthology.orchestra.run.smithi195.stdout:x86_64 2024-04-26T04:41:18.117 DEBUG:teuthology.orchestra.run.smithi195:> cat /etc/os-release 2024-04-26T04:41:18.174 INFO:teuthology.orchestra.run.smithi195.stdout:NAME="CentOS Stream" 2024-04-26T04:41:18.174 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION="9" 2024-04-26T04:41:18.174 INFO:teuthology.orchestra.run.smithi195.stdout:ID="centos" 2024-04-26T04:41:18.174 INFO:teuthology.orchestra.run.smithi195.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:41:18.174 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION_ID="9" 2024-04-26T04:41:18.174 INFO:teuthology.orchestra.run.smithi195.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:ANSI_COLOR="0;31" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:41:18.175 INFO:teuthology.orchestra.run.smithi195.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:41:18.175 INFO:teuthology.lock.ops:Updating smithi195.front.sepia.ceph.com on lock server 2024-04-26T04:41:18.199 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-26T04:41:18.209 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-26T04:41:18.234 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-26T04:41:18.234 DEBUG:teuthology.orchestra.run.smithi152:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:41:18.237 DEBUG:teuthology.orchestra.run.smithi195:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:41:18.253 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-26T04:41:18.258 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-26T04:41:18.259 DEBUG:teuthology.orchestra.run.smithi152:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:41:18.295 DEBUG:teuthology.orchestra.run.smithi195:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:41:18.401 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-26T04:41:18.455 INFO:teuthology.run_tasks:Running task kernel... 2024-04-26T04:41:18.470 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-26T04:41:18.470 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-26T04:41:18.470 DEBUG:teuthology.orchestra.run.smithi152:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:41:18.471 DEBUG:teuthology.orchestra.run.smithi195:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:41:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:41:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:41:18.489 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2024-04-26T04:41:18.489 DEBUG:teuthology.orchestra.run.smithi195:> uname -r 2024-04-26T04:41:18.545 INFO:teuthology.orchestra.run.smithi195.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:41:18.545 INFO:teuthology.task.kernel:Running kernel on smithi195: 5.14.0-437.el9.x86_64 2024-04-26T04:41:18.546 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum install -y kernel 2024-04-26T04:41:18.547 INFO:teuthology.orchestra.run.smithi152.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:41:18.547 INFO:teuthology.task.kernel:Running kernel on smithi152: 5.14.0-437.el9.x86_64 2024-04-26T04:41:18.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum install -y kernel 2024-04-26T04:41:20.716 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.0 MB 00:00 2024-04-26T04:41:20.819 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.0 MB 00:00 2024-04-26T04:41:22.717 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - AppStream 20 MB/s | 19 MB 00:00 2024-04-26T04:41:22.888 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 19 MB/s | 19 MB 00:00 2024-04-26T04:41:26.962 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.2 MB 00:00 2024-04-26T04:41:27.143 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.2 MB 00:00 2024-04-26T04:41:28.607 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - Extras packages 50 kB/s | 16 kB 00:00 2024-04-26T04:41:28.886 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 29 kB/s | 16 kB 00:00 2024-04-26T04:41:30.690 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-04-26T04:41:33.142 INFO:teuthology.orchestra.run.smithi195.stdout:Extra Packages for Enterprise Linux 4.7 MB/s | 21 MB 00:04 2024-04-26T04:41:35.778 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-26T04:41:37.677 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:41:37.677 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:41:37.767 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:41:37.768 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:41:37.768 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:41:37.871 DEBUG:teuthology.orchestra.run.smithi152:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:41:38.371 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:41:35 AM UTC. 2024-04-26T04:41:38.489 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:41:38.516 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout:Reinstalling: 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 255 k 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 0 2024-04-26T04:41:38.517 INFO:teuthology.orchestra.run.smithi152.stderr:Operation aborted. 2024-04-26T04:41:38.533 INFO:teuthology.orchestra.run.smithi195.stdout:lab-extras 4.2 kB/s | 1.7 kB 00:00 2024-04-26T04:41:38.578 INFO:teuthology.orchestra.run.smithi152.stdout:Is this ok [y/N]: 2024-04-26T04:41:38.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum reinstall -y kernel || true 2024-04-26T04:41:39.068 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:04 ago on Fri 26 Apr 2024 04:41:35 AM UTC. 2024-04-26T04:41:39.184 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:41:39.184 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:41:39.185 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:39.185 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:41:39.185 INFO:teuthology.orchestra.run.smithi152.stdout:Reinstalling: 2024-04-26T04:41:39.185 INFO:teuthology.orchestra.run.smithi152.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:39.185 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:41:39.185 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:41:39.185 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:41:39.186 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:41:39.186 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 255 k 2024-04-26T04:41:39.186 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 0 2024-04-26T04:41:39.186 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:41:39.448 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.8 MB/s | 255 kB 00:00 2024-04-26T04:41:39.448 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:41:39.448 INFO:teuthology.orchestra.run.smithi152.stdout:Total 966 kB/s | 255 kB 00:00 2024-04-26T04:41:39.467 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:41:39.660 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:41:39.660 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:41:39.665 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:41:39.665 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:41:39.805 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:41:39.882 INFO:teuthology.orchestra.run.smithi152.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:39.883 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:40.364 INFO:teuthology.orchestra.run.smithi195.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:41:40.365 INFO:teuthology.orchestra.run.smithi195.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:41:40.467 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:41:40.468 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:41:40.468 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:41:40.590 DEBUG:teuthology.orchestra.run.smithi195:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:41:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:41.091 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:41:38 AM UTC. 2024-04-26T04:41:41.228 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout:Reinstalling: 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:41:41.234 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:41:41.235 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 255 k 2024-04-26T04:41:41.235 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 0 2024-04-26T04:41:41.235 INFO:teuthology.orchestra.run.smithi195.stderr:Operation aborted. 2024-04-26T04:41:41.300 INFO:teuthology.orchestra.run.smithi195.stdout:Is this ok [y/N]: 2024-04-26T04:41:41.300 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum reinstall -y kernel || true 2024-04-26T04:41:41.362 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:41.363 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:41:41.363 INFO:teuthology.orchestra.run.smithi152.stdout:Reinstalled: 2024-04-26T04:41:41.363 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:41.363 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:41:41.363 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:41:41.543 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:41:41.589 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:41.590 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:41:41.590 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:41:41.590 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-26T04:41:41.590 DEBUG:teuthology.orchestra.run.smithi152:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:41:41.618 INFO:teuthology.orchestra.run.smithi152.stdout:ttyS1 2024-04-26T04:41:41.641 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:41.793 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:41:38 AM UTC. 2024-04-26T04:41:41.909 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:41:41.910 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:41:41.910 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:41.910 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:41:41.910 INFO:teuthology.orchestra.run.smithi195.stdout:Reinstalling: 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 255 k 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 0 2024-04-26T04:41:41.911 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:41:42.110 INFO:teuthology.orchestra.run.smithi195.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.9 MB/s | 255 kB 00:00 2024-04-26T04:41:42.111 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:41:42.111 INFO:teuthology.orchestra.run.smithi195.stdout:Total 1.2 MB/s | 255 kB 00:00 2024-04-26T04:41:42.129 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:41:42.315 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:41:42.315 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:41:42.320 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:41:42.320 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:41:42.456 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:41:42.545 INFO:teuthology.orchestra.run.smithi195.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:42.545 INFO:teuthology.orchestra.run.smithi195.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:43.185 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:43.185 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:43.945 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:43.945 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:41:43.945 INFO:teuthology.orchestra.run.smithi195.stdout:Reinstalled: 2024-04-26T04:41:43.945 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:43.946 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:41:43.946 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:41:44.115 DEBUG:teuthology.orchestra.run.smithi195:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:41:44.162 INFO:teuthology.orchestra.run.smithi195.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:44.162 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:41:44.163 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:41:44.163 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-26T04:41:44.163 DEBUG:teuthology.orchestra.run.smithi195:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:41:44.197 INFO:teuthology.orchestra.run.smithi195.stdout:ttyS1 2024-04-26T04:41:44.221 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:44.221 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-26T04:41:44.229 INFO:teuthology.task.internal:Creating test directory... 2024-04-26T04:41:44.229 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:41:44.231 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:41:44.276 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-26T04:41:44.282 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-26T04:41:44.289 INFO:teuthology.task.internal:Creating archive directory... 2024-04-26T04:41:44.289 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:41:44.292 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:41:44.346 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-26T04:41:44.353 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-26T04:41:44.353 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:41:44.369 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:41:44.399 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:44.409 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:44.421 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:44.430 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:44.432 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-26T04:41:44.439 INFO:teuthology.task.internal:Configuring sudo... 2024-04-26T04:41:44.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:41:44.454 DEBUG:teuthology.orchestra.run.smithi195:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:41:44.496 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-26T04:41:44.504 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-26T04:41:44.505 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:41:44.521 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:41:44.550 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:44.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:44.665 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:44.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:44.788 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:41:44.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:41:44.854 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:44.895 DEBUG:teuthology.orchestra.run.smithi195:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:44.948 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:45.029 DEBUG:teuthology.orchestra.run.smithi195:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:45.068 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:41:45.069 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:41:45.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo service rsyslog restart 2024-04-26T04:41:45.135 DEBUG:teuthology.orchestra.run.smithi195:> sudo service rsyslog restart 2024-04-26T04:41:45.168 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:41:45.206 INFO:teuthology.orchestra.run.smithi195.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:41:45.663 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-26T04:41:45.673 INFO:teuthology.task.internal:Starting timer... 2024-04-26T04:41:45.673 INFO:teuthology.run_tasks:Running task pcp... 2024-04-26T04:41:45.702 INFO:teuthology.run_tasks:Running task selinux... 2024-04-26T04:41:45.711 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-26T04:41:45.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo service auditd rotate 2024-04-26T04:41:45.772 INFO:teuthology.orchestra.run.smithi152.stdout:Rotating logs: 2024-04-26T04:41:45.774 DEBUG:teuthology.orchestra.run.smithi195:> sudo service auditd rotate 2024-04-26T04:41:45.857 INFO:teuthology.orchestra.run.smithi195.stdout:Rotating logs: 2024-04-26T04:41:45.859 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-26T04:41:45.859 DEBUG:teuthology.orchestra.run.smithi152:> /usr/sbin/getenforce 2024-04-26T04:41:45.884 INFO:teuthology.orchestra.run.smithi152.stdout:Permissive 2024-04-26T04:41:45.884 DEBUG:teuthology.orchestra.run.smithi195:> /usr/sbin/getenforce 2024-04-26T04:41:45.909 INFO:teuthology.orchestra.run.smithi195.stdout:Permissive 2024-04-26T04:41:45.910 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi152.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi195.front.sepia.ceph.com': 'permissive'} 2024-04-26T04:41:45.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:41:45.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:41:45.950 DEBUG:teuthology.orchestra.run.smithi195:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:41:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:41:45.975 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-26T04:41:45.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:41:46.016 DEBUG:teuthology.orchestra.run.smithi195:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:41:46.040 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-26T04:41:46.049 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-26T04:41:46.069 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-26T04:41:46.071 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi152.front.sepia.ceph.com,smithi195.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-26T04:45:15.429 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi152.front.sepia.ceph.com'), Remote(name='ubuntu@smithi195.front.sepia.ceph.com')] 2024-04-26T04:45:15.430 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-26T04:45:15.431 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:45:15.510 DEBUG:teuthology.orchestra.run.smithi152:> true 2024-04-26T04:45:15.597 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-26T04:45:15.597 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-04-26T04:45:15.598 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:45:15.677 DEBUG:teuthology.orchestra.run.smithi195:> true 2024-04-26T04:45:15.756 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-04-26T04:45:15.757 INFO:teuthology.run_tasks:Running task clock... 2024-04-26T04:45:15.768 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-26T04:45:15.768 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:45:15.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:45:15.771 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:45:15.771 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:45:15.796 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:45:15.809 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:45:15.824 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:45:15.833 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: ntpd: command not found 2024-04-26T04:45:15.837 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:45:15.843 INFO:teuthology.orchestra.run.smithi152.stdout:506 Cannot talk to daemon 2024-04-26T04:45:15.857 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:45:15.859 INFO:teuthology.orchestra.run.smithi195.stderr:sudo: ntpd: command not found 2024-04-26T04:45:15.869 INFO:teuthology.orchestra.run.smithi195.stdout:506 Cannot talk to daemon 2024-04-26T04:45:15.870 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:45:15.883 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:45:15.896 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:45:15.933 INFO:teuthology.orchestra.run.smithi152.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:45:15.936 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:45:15.936 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2024-04-26T04:45:15.937 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.937 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.937 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.937 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.957 INFO:teuthology.orchestra.run.smithi195.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:45:15.960 INFO:teuthology.orchestra.run.smithi195.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:45:15.960 INFO:teuthology.orchestra.run.smithi195.stdout:=============================================================================== 2024-04-26T04:45:15.960 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.960 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.960 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.960 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:15.961 INFO:teuthology.run_tasks:Running task pexec... 2024-04-26T04:45:15.971 INFO:teuthology.task.pexec:Executing custom commands... 2024-04-26T04:45:15.971 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi152.front.sepia.ceph.com 2024-04-26T04:45:15.971 DEBUG:teuthology.orchestra.run.smithi152:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-26T04:45:15.972 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi195.front.sepia.ceph.com 2024-04-26T04:45:15.972 DEBUG:teuthology.orchestra.run.smithi195:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-26T04:45:16.525 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:40 ago on Fri 26 Apr 2024 04:44:36 AM UTC. 2024-04-26T04:45:16.531 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:40 ago on Fri 26 Apr 2024 04:44:36 AM UTC. 2024-04-26T04:45:16.643 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:45:16.644 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:45:16.644 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-26T04:45:16.644 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:45:16.644 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:45:16.644 INFO:teuthology.orchestra.run.smithi152.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-04-26T04:45:16.644 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:45:16.644 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:45:16.645 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:45:16.645 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:45:16.645 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:45:16.645 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 3.1 M 2024-04-26T04:45:16.645 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 9.8 M 2024-04-26T04:45:16.645 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:45:16.647 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:45:16.648 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:45:16.649 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 3.1 M 2024-04-26T04:45:16.649 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 9.8 M 2024-04-26T04:45:16.649 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:45:17.123 INFO:teuthology.orchestra.run.smithi152.stdout:runc-1.1.12-2.el9.x86_64.rpm 15 MB/s | 3.1 MB 00:00 2024-04-26T04:45:17.124 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:17.124 INFO:teuthology.orchestra.run.smithi152.stdout:Total 6.5 MB/s | 3.1 MB 00:00 2024-04-26T04:45:17.160 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:45:17.166 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:45:17.166 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:45:17.249 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:45:17.249 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:45:17.468 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:45:17.566 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:45:18.113 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:45:18.397 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:45:18.398 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:45:18.398 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:45:18.398 INFO:teuthology.orchestra.run.smithi152.stdout: runc-4:1.1.12-2.el9.x86_64 2024-04-26T04:45:18.398 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:45:18.398 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:45:18.606 INFO:teuthology.orchestra.run.smithi195.stdout:runc-1.1.12-2.el9.x86_64.rpm 1.7 MB/s | 3.1 MB 00:01 2024-04-26T04:45:18.607 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:18.607 INFO:teuthology.orchestra.run.smithi195.stdout:Total 1.6 MB/s | 3.1 MB 00:01 2024-04-26T04:45:18.621 DEBUG:teuthology.parallel:result is None 2024-04-26T04:45:18.645 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:45:18.652 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:45:18.652 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:45:18.738 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:45:18.738 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:45:18.951 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:45:19.049 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:45:19.503 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:45:19.771 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:45:19.772 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:45:19.772 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:45:19.772 INFO:teuthology.orchestra.run.smithi195.stdout: runc-4:1.1.12-2.el9.x86_64 2024-04-26T04:45:19.772 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:45:19.772 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:45:20.032 DEBUG:teuthology.parallel:result is None 2024-04-26T04:45:20.032 INFO:teuthology.run_tasks:Running task install... 2024-04-26T04:45:20.041 DEBUG:teuthology.task.install:project ceph 2024-04-26T04:45:20.041 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'}} 2024-04-26T04:45:20.042 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:45:20.042 INFO:teuthology.task.install:Using flavor: default 2024-04-26T04:45:20.049 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-26T04:45:20.049 INFO:teuthology.task.install:extra packages: [] 2024-04-26T04:45:20.050 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:45:20.050 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:20.050 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:20.050 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:20.050 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:20.050 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:20.051 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:45:20.053 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:45:20.053 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:20.053 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:20.053 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:20.053 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:20.053 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:20.053 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:45:20.296 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:45:20.297 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:45:20.322 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:45:20.322 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:45:20.507 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:45:20.507 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:45:20.507 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:45:20.531 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:45:20.531 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:45:20.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:45:20.532 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:45:20.532 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:20.532 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:20.532 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:20.532 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:20.533 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:20.533 DEBUG:teuthology.orchestra.run.smithi195:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:45:20.557 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:45:20.557 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:20.557 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:20.558 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:20.558 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:20.558 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:20.558 DEBUG:teuthology.orchestra.run.smithi152:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:45:20.597 DEBUG:teuthology.orchestra.run.smithi195:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:45:20.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:45:20.672 DEBUG:teuthology.orchestra.run.smithi195:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:45:20.696 DEBUG:teuthology.orchestra.run.smithi152:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:45:20.738 INFO:teuthology.orchestra.run.smithi195.stdout:check_obsoletes = 1 2024-04-26T04:45:20.740 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum clean all 2024-04-26T04:45:20.761 INFO:teuthology.orchestra.run.smithi152.stdout:check_obsoletes = 1 2024-04-26T04:45:20.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum clean all 2024-04-26T04:45:20.972 INFO:teuthology.orchestra.run.smithi195.stdout:48 files removed 2024-04-26T04:45:20.988 INFO:teuthology.orchestra.run.smithi152.stdout:48 files removed 2024-04-26T04:45:21.006 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-radosgw 2024-04-26T04:45:21.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-radosgw 2024-04-26T04:45:21.714 INFO:teuthology.orchestra.run.smithi152.stdout:ceph packages for x86_64 178 kB/s | 77 kB 00:00 2024-04-26T04:45:21.715 INFO:teuthology.orchestra.run.smithi195.stdout:ceph packages for x86_64 175 kB/s | 77 kB 00:00 2024-04-26T04:45:22.307 INFO:teuthology.orchestra.run.smithi152.stdout:ceph noarch packages 19 kB/s | 11 kB 00:00 2024-04-26T04:45:22.325 INFO:teuthology.orchestra.run.smithi195.stdout:ceph noarch packages 19 kB/s | 11 kB 00:00 2024-04-26T04:45:22.770 INFO:teuthology.orchestra.run.smithi195.stdout:ceph source packages 4.4 kB/s | 1.8 kB 00:00 2024-04-26T04:45:22.815 INFO:teuthology.orchestra.run.smithi152.stdout:ceph source packages 3.8 kB/s | 1.8 kB 00:00 2024-04-26T04:45:24.949 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - BaseOS 3.8 MB/s | 8.0 MB 00:02 2024-04-26T04:45:25.419 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 3.1 MB/s | 8.0 MB 00:02 2024-04-26T04:45:28.803 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 8.2 MB/s | 19 MB 00:02 2024-04-26T04:45:29.466 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - AppStream 5.6 MB/s | 19 MB 00:03 2024-04-26T04:45:33.412 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 6.0 MB/s | 6.2 MB 00:01 2024-04-26T04:45:34.692 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 6.2 MB 00:01 2024-04-26T04:45:34.985 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 37 kB/s | 16 kB 00:00 2024-04-26T04:45:36.306 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 16 kB 00:00 2024-04-26T04:45:36.512 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 15 MB/s | 21 MB 00:01 2024-04-26T04:45:37.341 INFO:teuthology.orchestra.run.smithi195.stdout:Extra Packages for Enterprise Linux 23 MB/s | 21 MB 00:00 2024-04-26T04:45:41.828 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 8.3 kB/s | 1.7 kB 00:00 2024-04-26T04:45:42.366 INFO:teuthology.orchestra.run.smithi195.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-04-26T04:45:43.969 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrading: 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:45:43.971 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:45:43.972 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout:Install 22 Packages 2024-04-26T04:45:43.973 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrade 2 Packages 2024-04-26T04:45:43.974 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:45:43.974 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 58 M 2024-04-26T04:45:43.974 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:45:44.496 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:45:44.498 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:45:44.498 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:45:44.498 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:45:44.498 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:45:44.498 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:45:44.498 INFO:teuthology.orchestra.run.smithi195.stdout:Upgrading: 2024-04-26T04:45:44.498 INFO:teuthology.orchestra.run.smithi195.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:45:44.499 INFO:teuthology.orchestra.run.smithi195.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:45:44.500 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:45:44.501 INFO:teuthology.orchestra.run.smithi195.stdout:Install 22 Packages 2024-04-26T04:45:44.501 INFO:teuthology.orchestra.run.smithi195.stdout:Upgrade 2 Packages 2024-04-26T04:45:44.501 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:45:44.501 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 58 M 2024-04-26T04:45:44.501 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:45:45.180 INFO:teuthology.orchestra.run.smithi152.stdout:(1/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9.x 15 MB/s | 14 MB 00:00 2024-04-26T04:45:45.247 INFO:teuthology.orchestra.run.smithi152.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 378 kB/s | 25 kB 00:00 2024-04-26T04:45:45.338 INFO:teuthology.orchestra.run.smithi152.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 6.8 MB/s | 637 kB 00:00 2024-04-26T04:45:45.422 INFO:teuthology.orchestra.run.smithi152.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 5.4 MB/s | 457 kB 00:00 2024-04-26T04:45:45.914 INFO:teuthology.orchestra.run.smithi152.stdout:(5/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 10 MB/s | 4.9 MB 00:00 2024-04-26T04:45:45.989 INFO:teuthology.orchestra.run.smithi152.stdout:(6/24): python3-ceph-argparse-17.2.7-904.gd8c6b 602 kB/s | 45 kB 00:00 2024-04-26T04:45:46.065 INFO:teuthology.orchestra.run.smithi152.stdout:(7/24): python3-ceph-common-17.2.7-904.gd8c6b0a 1.5 MB/s | 117 kB 00:00 2024-04-26T04:45:46.132 INFO:teuthology.orchestra.run.smithi152.stdout:(8/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 2.3 MB/s | 156 kB 00:00 2024-04-26T04:45:46.199 INFO:teuthology.orchestra.run.smithi152.stdout:(9/24): python3-rados-17.2.7-904.gd8c6b0a3.el9. 4.7 MB/s | 320 kB 00:00 2024-04-26T04:45:46.244 INFO:teuthology.orchestra.run.smithi195.stdout:(1/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9.x 10 MB/s | 14 MB 00:01 2024-04-26T04:45:46.249 INFO:teuthology.orchestra.run.smithi152.stdout:(10/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 5.7 MB/s | 293 kB 00:00 2024-04-26T04:45:46.277 INFO:teuthology.orchestra.run.smithi195.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 763 kB/s | 25 kB 00:00 2024-04-26T04:45:46.307 INFO:teuthology.orchestra.run.smithi152.stdout:(11/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.7 MB/s | 100 kB 00:00 2024-04-26T04:45:46.344 INFO:teuthology.orchestra.run.smithi195.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 9.3 MB/s | 637 kB 00:00 2024-04-26T04:45:46.411 INFO:teuthology.orchestra.run.smithi195.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 6.7 MB/s | 457 kB 00:00 2024-04-26T04:45:46.458 INFO:teuthology.orchestra.run.smithi152.stdout:(12/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86 2.2 MB/s | 5.0 MB 00:02 2024-04-26T04:45:46.492 INFO:teuthology.orchestra.run.smithi152.stdout:(13/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 872 kB/s | 160 kB 00:00 2024-04-26T04:45:46.509 INFO:teuthology.orchestra.run.smithi152.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 901 kB/s | 45 kB 00:00 2024-04-26T04:45:46.553 INFO:teuthology.orchestra.run.smithi152.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 11 MB/s | 662 kB 00:00 2024-04-26T04:45:46.576 INFO:teuthology.orchestra.run.smithi152.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.3 MB/s | 292 kB 00:00 2024-04-26T04:45:46.593 INFO:teuthology.orchestra.run.smithi152.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-04-26T04:45:46.693 INFO:teuthology.orchestra.run.smithi152.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 487 kB/s | 49 kB 00:00 2024-04-26T04:45:46.752 INFO:teuthology.orchestra.run.smithi152.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.7 MB/s | 309 kB 00:00 2024-04-26T04:45:46.777 INFO:teuthology.orchestra.run.smithi152.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 805 kB/s | 67 kB 00:00 2024-04-26T04:45:46.778 INFO:teuthology.orchestra.run.smithi195.stdout:(5/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 13 MB/s | 4.9 MB 00:00 2024-04-26T04:45:46.836 INFO:teuthology.orchestra.run.smithi195.stdout:(6/24): python3-ceph-argparse-17.2.7-904.gd8c6b 775 kB/s | 45 kB 00:00 2024-04-26T04:45:46.887 INFO:teuthology.orchestra.run.smithi195.stdout:(7/24): python3-ceph-common-17.2.7-904.gd8c6b0a 2.3 MB/s | 117 kB 00:00 2024-04-26T04:45:46.954 INFO:teuthology.orchestra.run.smithi195.stdout:(8/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86_ 2.3 MB/s | 5.0 MB 00:02 2024-04-26T04:45:46.979 INFO:teuthology.orchestra.run.smithi195.stdout:(9/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 1.7 MB/s | 156 kB 00:00 2024-04-26T04:45:46.986 INFO:teuthology.orchestra.run.smithi152.stdout:(21/24): thrift-0.15.0-2.el9.x86_64.rpm 6.8 MB/s | 1.6 MB 00:00 2024-04-26T04:45:47.021 INFO:teuthology.orchestra.run.smithi195.stdout:(10/24): python3-rados-17.2.7-904.gd8c6b0a3.el9 4.7 MB/s | 320 kB 00:00 2024-04-26T04:45:47.046 INFO:teuthology.orchestra.run.smithi195.stdout:(11/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 4.3 MB/s | 293 kB 00:00 2024-04-26T04:45:47.105 INFO:teuthology.orchestra.run.smithi195.stdout:(12/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.2 MB/s | 100 kB 00:00 2024-04-26T04:45:47.172 INFO:teuthology.orchestra.run.smithi195.stdout:(13/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.2 MB/s | 160 kB 00:00 2024-04-26T04:45:47.197 INFO:teuthology.orchestra.run.smithi195.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 499 kB/s | 45 kB 00:00 2024-04-26T04:45:47.231 INFO:teuthology.orchestra.run.smithi195.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 11 MB/s | 662 kB 00:00 2024-04-26T04:45:47.256 INFO:teuthology.orchestra.run.smithi195.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.9 MB/s | 292 kB 00:00 2024-04-26T04:45:47.281 INFO:teuthology.orchestra.run.smithi195.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 850 kB/s | 42 kB 00:00 2024-04-26T04:45:47.311 INFO:teuthology.orchestra.run.smithi152.stdout:(22/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 9.0 MB/s | 2.9 MB 00:00 2024-04-26T04:45:47.390 INFO:teuthology.orchestra.run.smithi195.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 448 kB/s | 49 kB 00:00 2024-04-26T04:45:47.411 INFO:teuthology.orchestra.run.smithi152.stdout:(23/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 4.9 MB/s | 3.1 MB 00:00 2024-04-26T04:45:47.440 INFO:teuthology.orchestra.run.smithi195.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.6 MB/s | 309 kB 00:00 2024-04-26T04:45:47.465 INFO:teuthology.orchestra.run.smithi195.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 898 kB/s | 67 kB 00:00 2024-04-26T04:45:47.591 INFO:teuthology.orchestra.run.smithi195.stdout:(21/24): thrift-0.15.0-2.el9.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2024-04-26T04:45:47.983 INFO:teuthology.orchestra.run.smithi195.stdout:(22/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 7.5 MB/s | 2.9 MB 00:00 2024-04-26T04:45:48.125 INFO:teuthology.orchestra.run.smithi195.stdout:(23/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 4.7 MB/s | 3.1 MB 00:00 2024-04-26T04:45:48.162 INFO:teuthology.orchestra.run.smithi152.stdout:(24/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x 5.7 MB/s | 22 MB 00:03 2024-04-26T04:45:48.165 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:48.165 INFO:teuthology.orchestra.run.smithi152.stdout:Total 14 MB/s | 58 MB 00:04 2024-04-26T04:45:48.227 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:45:48.261 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:45:48.261 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:45:48.693 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:45:48.693 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:45:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:45:50.343 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:45:50.381 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:50.455 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:50.543 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:45:50.582 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:50.670 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:50.716 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:45:50.746 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:45:51.125 INFO:teuthology.orchestra.run.smithi195.stdout:(24/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x 3.6 MB/s | 22 MB 00:06 2024-04-26T04:45:51.128 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:51.128 INFO:teuthology.orchestra.run.smithi195.stdout:Total 8.7 MB/s | 58 MB 00:06 2024-04-26T04:45:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:45:51.189 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:45:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:51.222 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:45:51.223 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:45:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:51.319 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:45:51.362 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:45:51.398 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:45:51.469 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:45:51.497 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:51.545 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:51.585 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:45:51.614 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:45:51.647 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:45:51.660 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:45:51.660 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:45:51.939 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:45:51.985 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:52.074 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:52.149 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:45:52.179 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:45:52.882 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:45:53.230 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:45:53.327 INFO:teuthology.orchestra.run.smithi195.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:53.419 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:53.497 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:45:53.520 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:53.608 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:53.654 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:45:53.683 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:45:54.116 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:45:54.148 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:54.202 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:54.231 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:45:54.273 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:45:54.311 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:45:54.373 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:45:54.401 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:54.466 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:54.505 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:45:54.534 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:45:54.567 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:45:54.614 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:54.651 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:54.856 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:45:54.897 INFO:teuthology.orchestra.run.smithi195.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:54.986 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:55.061 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:45:55.091 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:45:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:55.294 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:45:55.294 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:45:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:45:55.323 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:45:57.575 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:57.629 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:58.137 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:58.191 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:58.276 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:58.276 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:45:58.276 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:45:58.276 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:45:58.309 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:46:04.858 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:46:04.858 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /sys 2024-04-26T04:46:04.858 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /proc 2024-04-26T04:46:04.858 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /mnt 2024-04-26T04:46:04.858 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /var/tmp 2024-04-26T04:46:04.859 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /home 2024-04-26T04:46:04.859 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /root 2024-04-26T04:46:04.859 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /tmp 2024-04-26T04:46:04.859 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:06.107 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:46:06.186 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:46:06.186 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:06.186 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:46:06.186 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:46:06.186 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:46:06.186 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:06.188 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:46:06.269 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:46:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:46:07.573 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:46:07.574 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:46:07.575 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:46:07.576 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:46:07.693 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:46:07.693 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /sys 2024-04-26T04:46:07.693 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /proc 2024-04-26T04:46:07.693 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /mnt 2024-04-26T04:46:07.693 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /var/tmp 2024-04-26T04:46:07.693 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /home 2024-04-26T04:46:07.693 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /root 2024-04-26T04:46:07.694 INFO:teuthology.orchestra.run.smithi195.stdout:skipping the directory /tmp 2024-04-26T04:46:07.694 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:08.270 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout:Upgraded: 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:46:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:46:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:08.273 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:08.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-test 2024-04-26T04:46:08.976 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:46:09.062 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:46:09.062 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:09.062 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:46:09.063 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:46:09.063 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:46:09.063 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:09.064 INFO:teuthology.orchestra.run.smithi195.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:46:09.090 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:28 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:09.146 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:46:09.146 INFO:teuthology.orchestra.run.smithi195.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:46:09.206 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:46:09.207 INFO:teuthology.orchestra.run.smithi152.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:46:09.208 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:46:09.208 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:09.208 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:46:09.208 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:09.208 INFO:teuthology.orchestra.run.smithi152.stdout:Install 4 Packages 2024-04-26T04:46:09.208 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:09.209 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 48 M 2024-04-26T04:46:09.209 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 192 M 2024-04-26T04:46:09.209 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:46:09.529 INFO:teuthology.orchestra.run.smithi152.stdout:(1/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 673 kB/s | 163 kB 00:00 2024-04-26T04:46:10.263 INFO:teuthology.orchestra.run.smithi152.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 87 kB/s | 64 kB 00:00 2024-04-26T04:46:10.389 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:46:10.389 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:46:10.389 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:46:10.389 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:46:10.390 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:46:10.390 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:46:10.390 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:46:10.390 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:46:10.390 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:46:10.390 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:46:10.391 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:46:10.391 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:46:10.393 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:46:10.393 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:46:10.393 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:46:10.393 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:46:10.393 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:46:10.393 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:46:10.394 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:46:10.394 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:46:10.394 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:46:10.394 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:46:10.394 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:46:10.395 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:46:10.395 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:46:10.395 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:46:10.446 INFO:teuthology.orchestra.run.smithi152.stdout:(3/4): socat-1.7.4.1-5.el9.x86_64.rpm 263 kB/s | 305 kB 00:01 2024-04-26T04:46:11.093 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:46:11.093 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:11.093 INFO:teuthology.orchestra.run.smithi195.stdout:Upgraded: 2024-04-26T04:46:11.093 INFO:teuthology.orchestra.run.smithi195.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.094 INFO:teuthology.orchestra.run.smithi195.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:11.095 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:11.377 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-test 2024-04-26T04:46:11.758 INFO:teuthology.orchestra.run.smithi152.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 19 MB/s | 48 MB 00:02 2024-04-26T04:46:11.758 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:11.759 INFO:teuthology.orchestra.run.smithi152.stdout:Total 19 MB/s | 48 MB 00:02 2024-04-26T04:46:11.785 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:46:11.806 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:46:11.806 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:46:11.889 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:29 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:12.008 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:12.010 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:12.010 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:46:12.010 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:12.010 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:46:12.010 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:46:12.010 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:46:12.011 INFO:teuthology.orchestra.run.smithi195.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:46:12.011 INFO:teuthology.orchestra.run.smithi195.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:46:12.011 INFO:teuthology.orchestra.run.smithi195.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:46:12.011 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:12.011 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:46:12.011 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:12.012 INFO:teuthology.orchestra.run.smithi195.stdout:Install 4 Packages 2024-04-26T04:46:12.012 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:12.012 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 48 M 2024-04-26T04:46:12.012 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 192 M 2024-04-26T04:46:12.012 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:46:12.168 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:46:12.169 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:46:12.642 INFO:teuthology.orchestra.run.smithi195.stdout:(1/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 640 kB/s | 163 kB 00:00 2024-04-26T04:46:12.694 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:46:12.763 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:46:12.803 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:46:12.826 INFO:teuthology.orchestra.run.smithi195.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 348 kB/s | 64 kB 00:00 2024-04-26T04:46:12.840 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:46:13.434 INFO:teuthology.orchestra.run.smithi195.stdout:(3/4): socat-1.7.4.1-5.el9.x86_64.rpm 291 kB/s | 305 kB 00:01 2024-04-26T04:46:14.726 INFO:teuthology.orchestra.run.smithi195.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 20 MB/s | 48 MB 00:02 2024-04-26T04:46:14.730 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:14.730 INFO:teuthology.orchestra.run.smithi195.stdout:Total 18 MB/s | 48 MB 00:02 2024-04-26T04:46:14.756 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:46:14.776 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:46:14.777 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:46:15.142 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:46:15.143 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:46:15.708 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:46:15.802 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:46:15.850 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:46:15.878 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:46:17.208 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:46:17.286 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:46:19.517 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:46:19.517 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:46:19.517 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:46:19.518 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:46:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:46:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:19.830 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:46:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:46:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:46:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:19.831 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:20.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph 2024-04-26T04:46:20.220 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:46:20.367 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:46:20.649 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:39 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:20.771 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:20.773 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:46:20.773 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:20.773 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:46:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:46:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout:Install 33 Packages 2024-04-26T04:46:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:20.777 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 27 M 2024-04-26T04:46:20.777 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 96 M 2024-04-26T04:46:20.777 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:46:21.524 INFO:teuthology.orchestra.run.smithi152.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 37 kB/s | 6.5 kB 00:00 2024-04-26T04:46:21.799 INFO:teuthology.orchestra.run.smithi152.stdout:(2/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 3.1 MB/s | 1.4 MB 00:00 2024-04-26T04:46:21.883 INFO:teuthology.orchestra.run.smithi152.stdout:(3/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 11 MB/s | 3.9 MB 00:00 2024-04-26T04:46:21.950 INFO:teuthology.orchestra.run.smithi152.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 3.5 MB/s | 240 kB 00:00 2024-04-26T04:46:22.100 INFO:teuthology.orchestra.run.smithi152.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.6 MB/s | 253 kB 00:00 2024-04-26T04:46:22.184 INFO:teuthology.orchestra.run.smithi152.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 15 MB/s | 1.2 MB 00:00 2024-04-26T04:46:22.217 INFO:teuthology.orchestra.run.smithi152.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 3.1 MB/s | 106 kB 00:00 2024-04-26T04:46:22.259 INFO:teuthology.orchestra.run.smithi152.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 3.2 MB/s | 135 kB 00:00 2024-04-26T04:46:22.301 INFO:teuthology.orchestra.run.smithi152.stdout:(9/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 2.2 MB/s | 2.1 MB 00:00 2024-04-26T04:46:22.326 INFO:teuthology.orchestra.run.smithi152.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 1.8 MB/s | 125 kB 00:00 2024-04-26T04:46:22.352 INFO:teuthology.orchestra.run.smithi152.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-04-26T04:46:22.369 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:46:22.369 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:46:22.369 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:46:22.369 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:46:22.377 INFO:teuthology.orchestra.run.smithi152.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 3.4 MB/s | 172 kB 00:00 2024-04-26T04:46:22.403 INFO:teuthology.orchestra.run.smithi152.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 694 kB/s | 35 kB 00:00 2024-04-26T04:46:22.428 INFO:teuthology.orchestra.run.smithi152.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-04-26T04:46:22.453 INFO:teuthology.orchestra.run.smithi152.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 831 kB/s | 42 kB 00:00 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:22.630 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:22.662 INFO:teuthology.orchestra.run.smithi152.stdout:(16/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_ 18 MB/s | 15 MB 00:00 2024-04-26T04:46:22.920 INFO:teuthology.orchestra.run.smithi152.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 88 kB/s | 43 kB 00:00 2024-04-26T04:46:22.958 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph 2024-04-26T04:46:23.021 INFO:teuthology.orchestra.run.smithi152.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 107 kB/s | 11 kB 00:00 2024-04-26T04:46:23.080 INFO:teuthology.orchestra.run.smithi152.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 274 kB/s | 172 kB 00:00 2024-04-26T04:46:23.129 INFO:teuthology.orchestra.run.smithi152.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 163 kB/s | 18 kB 00:00 2024-04-26T04:46:23.180 INFO:teuthology.orchestra.run.smithi152.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 231 kB/s | 23 kB 00:00 2024-04-26T04:46:23.238 INFO:teuthology.orchestra.run.smithi152.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 622 kB/s | 358 kB 00:00 2024-04-26T04:46:23.272 INFO:teuthology.orchestra.run.smithi152.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 136 kB/s | 19 kB 00:00 2024-04-26T04:46:23.297 INFO:teuthology.orchestra.run.smithi152.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 167 kB/s | 20 kB 00:00 2024-04-26T04:46:23.381 INFO:teuthology.orchestra.run.smithi152.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 728 kB/s | 79 kB 00:00 2024-04-26T04:46:23.406 INFO:teuthology.orchestra.run.smithi152.stdout:(26/33): python3-logutils-0.3.5-21.el9.noarch.r 346 kB/s | 46 kB 00:00 2024-04-26T04:46:23.465 INFO:teuthology.orchestra.run.smithi152.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 196 kB/s | 16 kB 00:00 2024-04-26T04:46:23.469 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:41 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:23.490 INFO:teuthology.orchestra.run.smithi152.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-04-26T04:46:23.557 INFO:teuthology.orchestra.run.smithi152.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 389 kB/s | 36 kB 00:00 2024-04-26T04:46:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:(30/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 511 kB/s | 90 kB 00:00 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:46:23.599 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:46:23.600 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:46:23.601 INFO:teuthology.orchestra.run.smithi195.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout:Install 33 Packages 2024-04-26T04:46:23.602 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:23.603 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 27 M 2024-04-26T04:46:23.603 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 96 M 2024-04-26T04:46:23.603 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:46:23.641 INFO:teuthology.orchestra.run.smithi152.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.5 MB/s | 230 kB 00:00 2024-04-26T04:46:23.683 INFO:teuthology.orchestra.run.smithi152.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 199 kB/s | 20 kB 00:00 2024-04-26T04:46:23.725 INFO:teuthology.orchestra.run.smithi152.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.5 MB/s | 427 kB 00:00 2024-04-26T04:46:23.729 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:23.729 INFO:teuthology.orchestra.run.smithi152.stdout:Total 9.2 MB/s | 27 MB 00:02 2024-04-26T04:46:23.848 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:46:23.885 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:46:23.886 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:46:24.154 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:46:24.155 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:46:24.318 INFO:teuthology.orchestra.run.smithi195.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 31 kB/s | 6.5 kB 00:00 2024-04-26T04:46:24.677 INFO:teuthology.orchestra.run.smithi195.stdout:(2/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 3.6 MB/s | 2.1 MB 00:00 2024-04-26T04:46:24.902 INFO:teuthology.orchestra.run.smithi195.stdout:(3/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 1.8 MB/s | 1.4 MB 00:00 2024-04-26T04:46:25.053 INFO:teuthology.orchestra.run.smithi195.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 1.6 MB/s | 240 kB 00:00 2024-04-26T04:46:25.169 INFO:teuthology.orchestra.run.smithi195.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.1 MB/s | 253 kB 00:00 2024-04-26T04:46:25.261 INFO:teuthology.orchestra.run.smithi195.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 13 MB/s | 1.2 MB 00:00 2024-04-26T04:46:25.291 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:46:25.320 INFO:teuthology.orchestra.run.smithi195.stdout:(7/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 3.9 MB/s | 3.9 MB 00:01 2024-04-26T04:46:25.345 INFO:teuthology.orchestra.run.smithi195.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 1.2 MB/s | 106 kB 00:00 2024-04-26T04:46:25.348 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:46:25.370 INFO:teuthology.orchestra.run.smithi195.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 2.6 MB/s | 135 kB 00:00 2024-04-26T04:46:25.385 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:46:25.395 INFO:teuthology.orchestra.run.smithi195.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 2.4 MB/s | 125 kB 00:00 2024-04-26T04:46:25.418 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:46:25.421 INFO:teuthology.orchestra.run.smithi195.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-04-26T04:46:25.446 INFO:teuthology.orchestra.run.smithi195.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 3.4 MB/s | 172 kB 00:00 2024-04-26T04:46:25.472 INFO:teuthology.orchestra.run.smithi195.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 695 kB/s | 35 kB 00:00 2024-04-26T04:46:25.497 INFO:teuthology.orchestra.run.smithi195.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-04-26T04:46:25.510 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:46:25.522 INFO:teuthology.orchestra.run.smithi195.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 829 kB/s | 42 kB 00:00 2024-04-26T04:46:25.597 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:46:25.637 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:46:25.673 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:46:25.703 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:46:25.738 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:46:25.756 INFO:teuthology.orchestra.run.smithi195.stdout:(16/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_ 14 MB/s | 15 MB 00:01 2024-04-26T04:46:25.771 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:46:25.803 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:46:25.838 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:46:25.881 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:46:25.919 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:46:25.992 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:46:26.073 INFO:teuthology.orchestra.run.smithi195.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 75 kB/s | 43 kB 00:00 2024-04-26T04:46:26.131 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:46:26.181 INFO:teuthology.orchestra.run.smithi195.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 98 kB/s | 11 kB 00:00 2024-04-26T04:46:26.210 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:46:26.264 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:46:26.282 INFO:teuthology.orchestra.run.smithi195.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 226 kB/s | 172 kB 00:00 2024-04-26T04:46:26.307 INFO:teuthology.orchestra.run.smithi195.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 141 kB/s | 18 kB 00:00 2024-04-26T04:46:26.311 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:46:26.368 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:46:26.390 INFO:teuthology.orchestra.run.smithi195.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 213 kB/s | 23 kB 00:00 2024-04-26T04:46:26.424 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:46:26.424 INFO:teuthology.orchestra.run.smithi195.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 578 kB/s | 358 kB 00:00 2024-04-26T04:46:26.450 INFO:teuthology.orchestra.run.smithi195.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 136 kB/s | 19 kB 00:00 2024-04-26T04:46:26.491 INFO:teuthology.orchestra.run.smithi195.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 195 kB/s | 20 kB 00:00 2024-04-26T04:46:26.559 INFO:teuthology.orchestra.run.smithi195.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 726 kB/s | 79 kB 00:00 2024-04-26T04:46:26.583 INFO:teuthology.orchestra.run.smithi195.stdout:(26/33): python3-logutils-0.3.5-21.el9.noarch.r 291 kB/s | 46 kB 00:00 2024-04-26T04:46:26.611 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:46:26.667 INFO:teuthology.orchestra.run.smithi195.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 151 kB/s | 16 kB 00:00 2024-04-26T04:46:26.674 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:46:26.692 INFO:teuthology.orchestra.run.smithi195.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.3 MB/s | 272 kB 00:00 2024-04-26T04:46:26.718 INFO:teuthology.orchestra.run.smithi195.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 671 kB/s | 90 kB 00:00 2024-04-26T04:46:26.752 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:46:26.801 INFO:teuthology.orchestra.run.smithi195.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 268 kB/s | 36 kB 00:00 2024-04-26T04:46:26.826 INFO:teuthology.orchestra.run.smithi195.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.7 MB/s | 230 kB 00:00 2024-04-26T04:46:26.841 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:46:26.882 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:46:26.885 INFO:teuthology.orchestra.run.smithi195.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.5 MB/s | 427 kB 00:00 2024-04-26T04:46:26.918 INFO:teuthology.orchestra.run.smithi195.stdout:(33/33): python3-zc-lockfile-2.0-10.el9.noarch. 171 kB/s | 20 kB 00:00 2024-04-26T04:46:26.924 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:26.924 INFO:teuthology.orchestra.run.smithi195.stdout:Total 8.2 MB/s | 27 MB 00:03 2024-04-26T04:46:26.997 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:46:27.038 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:46:27.075 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:46:27.075 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:46:27.146 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:46:27.177 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:27.201 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:27.201 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:27.201 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:46:27.202 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:27.202 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:27.202 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:27.341 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:46:27.341 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:46:28.509 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:46:28.597 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:46:28.611 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:28.636 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:46:28.639 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:28.639 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:28.639 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:46:28.640 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:28.640 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:28.640 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:28.670 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:46:28.760 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:46:28.847 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:46:28.888 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:46:28.924 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:46:28.954 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:46:28.989 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:46:29.022 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:46:29.029 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:29.052 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:29.052 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:29.052 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:46:29.052 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:29.053 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:29.053 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:29.062 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:46:29.097 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:46:29.132 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:46:29.170 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:46:29.242 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:46:29.254 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:29.278 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:29.278 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:29.278 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:46:29.278 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:29.278 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:29.278 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:29.342 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:29.379 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:46:29.468 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:46:29.523 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:46:29.561 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:46:29.610 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:46:29.665 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:46:29.849 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:46:29.899 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:46:29.976 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:46:30.064 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:46:30.106 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:46:30.220 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:46:30.385 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:46:30.417 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:30.441 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:30.441 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:30.441 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:46:30.441 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:30.441 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:30.441 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:46:30.862 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:46:30.863 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:46:30.864 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:46:30.865 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:46:30.865 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:46:30.865 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:46:30.865 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:46:30.865 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:46:30.865 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:46:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:46:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:31.725 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:31.810 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:31.837 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:31.837 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:31.837 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:46:31.838 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:31.838 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:31.838 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:32.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-base 2024-04-26T04:46:32.211 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:32.233 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:32.234 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:32.234 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:46:32.234 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:32.234 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:32.234 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:32.445 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:32.468 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:32.468 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:32.468 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:46:32.468 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:32.468 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:32.468 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:32.532 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:32.614 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:51 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:32.693 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:32.734 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:32.735 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:46:32.735 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:32.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install cephadm 2024-04-26T04:46:33.311 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:52 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:33.429 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:33.430 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:46:33.431 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:33.431 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 82 k 2024-04-26T04:46:33.431 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 372 k 2024-04-26T04:46:33.431 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:46:33.756 INFO:teuthology.orchestra.run.smithi152.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 252 kB/s | 82 kB 00:00 2024-04-26T04:46:33.756 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:33.757 INFO:teuthology.orchestra.run.smithi152.stdout:Total 251 kB/s | 82 kB 00:00 2024-04-26T04:46:33.757 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:46:33.762 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:46:33.762 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:46:33.764 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:46:33.764 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:46:33.845 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:46:34.089 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:46:34.090 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:46:34.091 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:46:34.092 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:46:34.092 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:46:34.093 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:46:34.093 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:46:34.093 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:46:34.093 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:46:34.093 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:46:34.093 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:46:34.297 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:34.409 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:34.758 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:34.974 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:46:34.975 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:46:34.976 INFO:teuthology.orchestra.run.smithi195.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:46:34.977 INFO:teuthology.orchestra.run.smithi195.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:46:34.977 INFO:teuthology.orchestra.run.smithi195.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:46:34.977 INFO:teuthology.orchestra.run.smithi195.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:46:34.977 INFO:teuthology.orchestra.run.smithi195.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:46:34.977 INFO:teuthology.orchestra.run.smithi195.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:46:34.977 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:34.977 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:35.021 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:35.021 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:35.021 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:46:35.021 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:35.021 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:35.021 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:35.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:46:35.334 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-base 2024-04-26T04:46:35.734 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:54 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:35.852 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:53 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:35.901 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:35.901 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:35.901 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-26T04:46:35.901 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:35.901 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:46:35.901 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 137 k 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 379 k 2024-04-26T04:46:35.902 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:46:35.930 INFO:teuthology.orchestra.run.smithi195.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:35.980 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:35.981 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:46:35.981 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:36.039 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install cephadm 2024-04-26T04:46:36.081 INFO:teuthology.orchestra.run.smithi152.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 623 kB/s | 137 kB 00:00 2024-04-26T04:46:36.082 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:36.082 INFO:teuthology.orchestra.run.smithi152.stdout:Total 617 kB/s | 137 kB 00:00 2024-04-26T04:46:36.082 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:46:36.090 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:46:36.090 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:46:36.147 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:46:36.147 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:46:36.299 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:46:36.386 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:36.407 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:36.408 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:46:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:36.549 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:54 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:36.668 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:36.669 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:36.669 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:46:36.669 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:36.669 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:46:36.669 INFO:teuthology.orchestra.run.smithi195.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:46:36.669 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:36.670 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:46:36.670 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:36.670 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:46:36.670 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:36.670 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 82 k 2024-04-26T04:46:36.670 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 372 k 2024-04-26T04:46:36.670 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:46:36.977 INFO:teuthology.orchestra.run.smithi195.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 267 kB/s | 82 kB 00:00 2024-04-26T04:46:36.978 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:36.978 INFO:teuthology.orchestra.run.smithi195.stdout:Total 266 kB/s | 82 kB 00:00 2024-04-26T04:46:36.978 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:46:36.983 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:46:36.983 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:46:36.985 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:46:36.985 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:46:37.062 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:46:37.311 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:37.311 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:37.311 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:46:37.311 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:37.311 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:37.311 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:37.507 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:37.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr 2024-04-26T04:46:37.605 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:37.973 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:38.034 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:57 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:38.115 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:38.158 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:38.159 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:46:38.159 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:38.193 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:38.193 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:38.193 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:46:38.193 INFO:teuthology.orchestra.run.smithi195.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:38.193 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:38.193 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:38.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:46:38.406 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:46:38.737 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:57 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:38.859 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:38.859 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout:Installing weak dependencies: 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:46:38.860 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:38.861 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:46:38.861 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:38.861 INFO:teuthology.orchestra.run.smithi152.stdout:Install 7 Packages 2024-04-26T04:46:38.861 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:38.861 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 3.7 M 2024-04-26T04:46:38.861 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 80 M 2024-04-26T04:46:38.861 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:46:38.921 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:56 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:39.043 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repo Size 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:46:39.044 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:39.045 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 137 k 2024-04-26T04:46:39.045 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 379 k 2024-04-26T04:46:39.045 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:46:39.259 INFO:teuthology.orchestra.run.smithi195.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 643 kB/s | 137 kB 00:00 2024-04-26T04:46:39.259 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:39.260 INFO:teuthology.orchestra.run.smithi195.stdout:Total 638 kB/s | 137 kB 00:00 2024-04-26T04:46:39.260 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:46:39.267 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:46:39.267 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:46:39.303 INFO:teuthology.orchestra.run.smithi152.stdout:(1/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 120 kB/s | 24 kB 00:00 2024-04-26T04:46:39.322 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:46:39.323 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:46:39.411 INFO:teuthology.orchestra.run.smithi152.stdout:(2/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 47 kB/s | 15 kB 00:00 2024-04-26T04:46:39.466 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:46:39.543 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:39.545 INFO:teuthology.orchestra.run.smithi152.stdout:(3/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 7.5 MB/s | 3.4 MB 00:00 2024-04-26T04:46:39.568 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:39.568 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:39.568 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:46:39.568 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:39.570 INFO:teuthology.orchestra.run.smithi152.stdout:(4/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 34 kB/s | 9.0 kB 00:00 2024-04-26T04:46:39.754 INFO:teuthology.orchestra.run.smithi152.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 147 kB/s | 31 kB 00:00 2024-04-26T04:46:39.837 INFO:teuthology.orchestra.run.smithi152.stdout:(6/7): python3-jwt-2.4.0-1.el9.noarch.rpm 96 kB/s | 41 kB 00:00 2024-04-26T04:46:40.321 INFO:teuthology.orchestra.run.smithi152.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 251 kB/s | 188 kB 00:00 2024-04-26T04:46:40.323 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:40.323 INFO:teuthology.orchestra.run.smithi152.stdout:Total 2.5 MB/s | 3.7 MB 00:01 2024-04-26T04:46:40.324 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:46:40.333 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:46:40.333 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:46:40.400 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:46:40.400 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:46:40.441 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:40.442 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:40.442 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:46:40.442 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:40.442 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:40.442 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:40.644 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-mgr 2024-04-26T04:46:40.706 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:46:40.780 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:46:40.816 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:46:40.877 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:46:40.910 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:46:40.939 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:46:41.166 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:41.245 INFO:teuthology.orchestra.run.smithi195.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:41.289 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:41.290 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:46:41.290 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:41.372 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:46:41.889 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:46:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:42.012 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:46:42.013 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout:Installing weak dependencies: 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout:Install 7 Packages 2024-04-26T04:46:42.014 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:42.015 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 3.7 M 2024-04-26T04:46:42.015 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 80 M 2024-04-26T04:46:42.015 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:46:42.304 INFO:teuthology.orchestra.run.smithi195.stdout:(1/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 74 kB/s | 15 kB 00:00 2024-04-26T04:46:42.329 INFO:teuthology.orchestra.run.smithi195.stdout:(2/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 110 kB/s | 24 kB 00:00 2024-04-26T04:46:42.396 INFO:teuthology.orchestra.run.smithi195.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 98 kB/s | 9.0 kB 00:00 2024-04-26T04:46:42.421 INFO:teuthology.orchestra.run.smithi195.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 446 kB/s | 41 kB 00:00 2024-04-26T04:46:42.440 INFO:teuthology.orchestra.run.smithi195.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 735 kB/s | 31 kB 00:00 2024-04-26T04:46:42.480 INFO:teuthology.orchestra.run.smithi195.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.2 MB/s | 188 kB 00:00 2024-04-26T04:46:42.730 INFO:teuthology.orchestra.run.smithi195.stdout:(7/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 5.4 MB/s | 3.4 MB 00:00 2024-04-26T04:46:42.734 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:42.734 INFO:teuthology.orchestra.run.smithi195.stdout:Total 5.1 MB/s | 3.7 MB 00:00 2024-04-26T04:46:42.734 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:46:42.741 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:46:42.741 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:46:42.801 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:46:42.802 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:46:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:46:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:46:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:46:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:46:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:46:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:46:43.101 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:46:43.157 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:46:43.191 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:46:43.237 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:46:43.269 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:46:43.298 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:46:43.311 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:46:43.311 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:43.311 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:46:43.311 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:43.311 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:43.311 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:43.311 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:46:43.312 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:46:43.312 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:46:43.312 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:46:43.312 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:43.312 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:43.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:46:44.014 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:03 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:44.134 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:44.135 INFO:teuthology.orchestra.run.smithi152.stdout:====================================================================================== 2024-04-26T04:46:44.135 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:46:44.135 INFO:teuthology.orchestra.run.smithi152.stdout:====================================================================================== 2024-04-26T04:46:44.135 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:46:44.135 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:46:44.135 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:46:44.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:46:44.137 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:44.137 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:46:44.137 INFO:teuthology.orchestra.run.smithi152.stdout:====================================================================================== 2024-04-26T04:46:44.137 INFO:teuthology.orchestra.run.smithi152.stdout:Install 12 Packages 2024-04-26T04:46:44.137 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:44.138 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 38 M 2024-04-26T04:46:44.138 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 209 M 2024-04-26T04:46:44.138 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:46:44.298 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:46:44.326 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:44.505 INFO:teuthology.orchestra.run.smithi152.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.0 MB/s | 195 kB 00:00 2024-04-26T04:46:44.547 INFO:teuthology.orchestra.run.smithi152.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 3.5 MB/s | 803 kB 00:00 2024-04-26T04:46:44.572 INFO:teuthology.orchestra.run.smithi152.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 493 kB/s | 33 kB 00:00 2024-04-26T04:46:44.622 INFO:teuthology.orchestra.run.smithi152.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 367 kB/s | 18 kB 00:00 2024-04-26T04:46:44.714 INFO:teuthology.orchestra.run.smithi152.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 18 MB/s | 3.0 MB 00:00 2024-04-26T04:46:44.739 INFO:teuthology.orchestra.run.smithi152.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 306 kB/s | 35 kB 00:00 2024-04-26T04:46:44.807 INFO:teuthology.orchestra.run.smithi152.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 3.6 MB/s | 244 kB 00:00 2024-04-26T04:46:44.915 INFO:teuthology.orchestra.run.smithi152.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 24 MB/s | 4.7 MB 00:00 2024-04-26T04:46:45.099 INFO:teuthology.orchestra.run.smithi152.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 18 MB/s | 5.1 MB 00:00 2024-04-26T04:46:45.124 INFO:teuthology.orchestra.run.smithi152.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.2 MB/s | 250 kB 00:00 2024-04-26T04:46:45.333 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:45.333 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:46:45.333 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:46:45.333 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:46:45.333 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:46:45.333 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:46:45.333 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:46:45.578 INFO:teuthology.orchestra.run.smithi195.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:46:45.579 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:45.579 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:45.708 INFO:teuthology.orchestra.run.smithi152.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 27 MB/s | 16 MB 00:00 2024-04-26T04:46:45.776 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:46:46.284 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:04 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:46.405 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:46.406 INFO:teuthology.orchestra.run.smithi195.stdout:====================================================================================== 2024-04-26T04:46:46.406 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout:====================================================================================== 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:46:46.407 INFO:teuthology.orchestra.run.smithi195.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout:====================================================================================== 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout:Install 12 Packages 2024-04-26T04:46:46.408 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:46.409 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 38 M 2024-04-26T04:46:46.409 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 209 M 2024-04-26T04:46:46.409 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:46:46.889 INFO:teuthology.orchestra.run.smithi195.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.3 MB/s | 195 kB 00:00 2024-04-26T04:46:46.924 INFO:teuthology.orchestra.run.smithi195.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 986 kB/s | 33 kB 00:00 2024-04-26T04:46:47.056 INFO:teuthology.orchestra.run.smithi195.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 27 MB/s | 3.0 MB 00:00 2024-04-26T04:46:47.089 INFO:teuthology.orchestra.run.smithi195.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 550 kB/s | 18 kB 00:00 2024-04-26T04:46:47.122 INFO:teuthology.orchestra.run.smithi195.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 1.0 MB/s | 35 kB 00:00 2024-04-26T04:46:47.174 INFO:teuthology.orchestra.run.smithi152.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-9 2.6 MB/s | 7.4 MB 00:02 2024-04-26T04:46:47.178 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:47.178 INFO:teuthology.orchestra.run.smithi152.stdout:Total 12 MB/s | 38 MB 00:03 2024-04-26T04:46:47.532 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:46:47.559 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:46:47.559 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:46:47.966 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:46:47.966 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:46:48.089 INFO:teuthology.orchestra.run.smithi195.stdout:(6/12): ceph-mgr-diskprediction-local-17.2.7-90 5.5 MB/s | 7.4 MB 00:01 2024-04-26T04:46:48.248 INFO:teuthology.orchestra.run.smithi195.stdout:(7/12): libgfortran-11.4.1-3.el9.x86_64.rpm 534 kB/s | 803 kB 00:01 2024-04-26T04:46:48.548 INFO:teuthology.orchestra.run.smithi195.stdout:(8/12): python3-devel-3.9.18-3.el9.x86_64.rpm 533 kB/s | 244 kB 00:00 2024-04-26T04:46:48.690 INFO:teuthology.orchestra.run.smithi195.stdout:(9/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 3.0 MB/s | 4.7 MB 00:01 2024-04-26T04:46:48.731 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:46:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:46:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:48.941 INFO:teuthology.orchestra.run.smithi195.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 7.4 MB/s | 5.1 MB 00:00 2024-04-26T04:46:48.966 INFO:teuthology.orchestra.run.smithi195.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 601 kB/s | 250 kB 00:00 2024-04-26T04:46:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:46:49.195 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:46:49.223 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:46:49.300 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:46:49.849 INFO:teuthology.orchestra.run.smithi195.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 14 MB/s | 16 MB 00:01 2024-04-26T04:46:49.852 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:49.852 INFO:teuthology.orchestra.run.smithi195.stdout:Total 11 MB/s | 38 MB 00:03 2024-04-26T04:46:49.952 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:46:50.087 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:46:50.198 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:46:50.223 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:46:50.223 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:46:50.631 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:46:50.631 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:46:51.329 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:51.370 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:46:51.429 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:46:51.472 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:51.570 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:51.615 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:46:51.792 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:46:51.822 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:46:51.898 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:46:52.366 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:46:52.392 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:52.550 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:46:52.676 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:46:53.897 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:46:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:46:54.876 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:54.876 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:46:54.929 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:46:54.972 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:46:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:55.295 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:46:55.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:46:56.202 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:15 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:46:56.323 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:46:56.324 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:56.324 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:46:56.325 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout:Install 13 Packages 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:46:56.326 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.2 M 2024-04-26T04:46:56.327 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 26 M 2024-04-26T04:46:56.327 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:46:57.203 INFO:teuthology.orchestra.run.smithi152.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 105 kB/s | 19 kB 00:00 2024-04-26T04:46:57.228 INFO:teuthology.orchestra.run.smithi152.stdout:(2/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 229 kB/s | 47 kB 00:00 2024-04-26T04:46:57.328 INFO:teuthology.orchestra.run.smithi152.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.7 MB/s | 222 kB 00:00 2024-04-26T04:46:57.354 INFO:teuthology.orchestra.run.smithi152.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.5 MB/s | 159 kB 00:00 2024-04-26T04:46:57.404 INFO:teuthology.orchestra.run.smithi152.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 3.6 MB/s | 279 kB 00:00 2024-04-26T04:46:57.429 INFO:teuthology.orchestra.run.smithi152.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 710 kB/s | 54 kB 00:00 2024-04-26T04:46:57.596 INFO:teuthology.orchestra.run.smithi152.stdout:(7/13): python3-certifi-2021.10.8-2.el9.noarch. 90 kB/s | 15 kB 00:00 2024-04-26T04:46:57.621 INFO:teuthology.orchestra.run.smithi152.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 148 kB/s | 32 kB 00:00 2024-04-26T04:46:57.647 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:57.647 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:46:57.647 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:57.647 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:57.647 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:46:57.647 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:46:57.648 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:46:57.648 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:46:57.648 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:46:57.648 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:46:57.648 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:57.648 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:46:57.697 INFO:teuthology.orchestra.run.smithi152.stdout:(9/13): python3-jsonpatch-1.21-16.el9.noarch.rp 38 kB/s | 26 kB 00:00 2024-04-26T04:46:57.856 INFO:teuthology.orchestra.run.smithi152.stdout:(10/13): python3-google-auth-2.29.0-1.el9.noarc 793 kB/s | 206 kB 00:00 2024-04-26T04:46:57.923 INFO:teuthology.orchestra.run.smithi152.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 262 kB/s | 59 kB 00:00 2024-04-26T04:46:58.006 INFO:teuthology.orchestra.run.smithi152.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 597 kB/s | 90 kB 00:00 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:46:58.020 INFO:teuthology.orchestra.run.smithi195.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:46:58.021 INFO:teuthology.orchestra.run.smithi195.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:46:58.021 INFO:teuthology.orchestra.run.smithi195.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:46:58.021 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:58.021 INFO:teuthology.orchestra.run.smithi195.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:58.021 INFO:teuthology.orchestra.run.smithi195.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:46:58.021 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:58.021 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:46:58.298 INFO:teuthology.orchestra.run.smithi152.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 1.5 MB/s | 1.0 MB 00:00 2024-04-26T04:46:58.299 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:58.300 INFO:teuthology.orchestra.run.smithi152.stdout:Total 1.1 MB/s | 2.2 MB 00:01 2024-04-26T04:46:58.364 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:46:58.379 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:46:58.379 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:46:58.386 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:46:58.462 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:46:58.463 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:46:58.903 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:16 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:46:59.023 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:46:59.024 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:46:59.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout:Install 13 Packages 2024-04-26T04:46:59.026 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:46:59.027 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 2.2 M 2024-04-26T04:46:59.027 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 26 M 2024-04-26T04:46:59.027 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:46:59.067 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:46:59.239 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:46:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:46:59.355 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:46:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:46:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:46:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:46:59.631 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:46:59.702 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:46:59.722 INFO:teuthology.orchestra.run.smithi195.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 113 kB/s | 26 kB 00:00 2024-04-26T04:46:59.748 INFO:teuthology.orchestra.run.smithi195.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 76 kB/s | 19 kB 00:00 2024-04-26T04:46:59.865 INFO:teuthology.orchestra.run.smithi195.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-04-26T04:46:59.890 INFO:teuthology.orchestra.run.smithi195.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 159 kB 00:00 2024-04-26T04:46:59.915 INFO:teuthology.orchestra.run.smithi195.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 5.4 MB/s | 279 kB 00:00 2024-04-26T04:46:59.941 INFO:teuthology.orchestra.run.smithi195.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-04-26T04:46:59.966 INFO:teuthology.orchestra.run.smithi195.stdout:(7/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 101 kB/s | 47 kB 00:00 2024-04-26T04:47:00.091 INFO:teuthology.orchestra.run.smithi195.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 183 kB/s | 32 kB 00:00 2024-04-26T04:47:00.116 INFO:teuthology.orchestra.run.smithi195.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 85 kB/s | 15 kB 00:00 2024-04-26T04:47:00.192 INFO:teuthology.orchestra.run.smithi195.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 791 kB/s | 59 kB 00:00 2024-04-26T04:47:00.217 INFO:teuthology.orchestra.run.smithi195.stdout:(11/13): python3-google-auth-2.29.0-1.el9.noarc 1.0 MB/s | 206 kB 00:00 2024-04-26T04:47:00.250 INFO:teuthology.orchestra.run.smithi195.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 6.5 MB/s | 1.0 MB 00:00 2024-04-26T04:47:00.276 INFO:teuthology.orchestra.run.smithi195.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 1.1 MB/s | 90 kB 00:00 2024-04-26T04:47:00.280 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:00.281 INFO:teuthology.orchestra.run.smithi195.stdout:Total 1.8 MB/s | 2.2 MB 00:01 2024-04-26T04:47:00.341 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:00.356 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:00.356 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:00.422 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:47:00.440 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:00.440 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:00.475 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:47:00.509 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:47:00.554 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:47:00.579 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:47:01.009 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:01.181 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:47:01.235 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:47:01.299 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:47:01.309 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:47:01.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:47:01.311 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:47:01.311 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:47:01.336 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:47:01.378 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:47:01.471 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:47:01.581 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:47:01.637 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:47:01.642 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:01.643 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:01.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:47:02.350 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:47:02.396 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:21 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:02.410 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:47:02.435 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:47:02.473 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:47:02.497 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:47:02.517 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:47:02.518 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:47:02.519 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:47:02.519 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:02.519 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:47:02.519 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:02.519 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2024-04-26T04:47:02.519 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:02.519 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 7.0 M 2024-04-26T04:47:02.520 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 33 M 2024-04-26T04:47:02.520 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:47:03.208 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:47:03.209 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:47:03.209 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:47:03.294 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.1 MB/s | 248 kB 00:00 2024-04-26T04:47:03.319 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 492 kB/s | 123 kB 00:00 2024-04-26T04:47:03.525 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:47:03.525 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:47:03.526 INFO:teuthology.orchestra.run.smithi195.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:47:03.527 INFO:teuthology.orchestra.run.smithi195.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:47:03.527 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:03.527 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:03.661 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 169 kB/s | 58 kB 00:00 2024-04-26T04:47:03.749 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:47:03.761 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 509 kB/s | 51 kB 00:00 2024-04-26T04:47:03.936 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 853 kB/s | 548 kB 00:00 2024-04-26T04:47:04.263 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:22 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:04.382 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:47:04.383 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:04.384 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:47:04.384 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:04.384 INFO:teuthology.orchestra.run.smithi195.stdout:Install 6 Packages 2024-04-26T04:47:04.384 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:04.384 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 7.0 M 2024-04-26T04:47:04.385 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 33 M 2024-04-26T04:47:04.385 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:47:04.895 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.3 MB/s | 6.0 MB 00:01 2024-04-26T04:47:04.897 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:04.897 INFO:teuthology.orchestra.run.smithi152.stdout:Total 2.9 MB/s | 7.0 MB 00:02 2024-04-26T04:47:04.970 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:47:04.982 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:47:04.982 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:47:05.062 INFO:teuthology.orchestra.run.smithi195.stdout:(1/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 485 kB/s | 123 kB 00:00 2024-04-26T04:47:05.078 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:47:05.078 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:47:05.396 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:47:05.549 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:47:05.604 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:47:05.678 INFO:teuthology.orchestra.run.smithi195.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 889 kB/s | 548 kB 00:00 2024-04-26T04:47:05.779 INFO:teuthology.orchestra.run.smithi195.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 578 kB/s | 58 kB 00:00 2024-04-26T04:47:05.887 INFO:teuthology.orchestra.run.smithi195.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 471 kB/s | 51 kB 00:00 2024-04-26T04:47:05.912 INFO:teuthology.orchestra.run.smithi195.stdout:(5/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 224 kB/s | 248 kB 00:01 2024-04-26T04:47:06.115 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:47:06.198 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:47:06.254 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:47:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:47:06.571 INFO:teuthology.orchestra.run.smithi195.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.4 MB/s | 6.0 MB 00:01 2024-04-26T04:47:06.628 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:06.628 INFO:teuthology.orchestra.run.smithi195.stdout:Total 3.2 MB/s | 7.0 MB 00:02 2024-04-26T04:47:06.647 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:06.659 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:06.660 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:06.756 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:06.757 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:07.065 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:07.082 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:47:07.082 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:47:07.082 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:47:07.082 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:47:07.083 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:47:07.083 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:47:07.240 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:47:07.298 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:47:07.364 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:07.365 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:07.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-fuse 2024-04-26T04:47:07.834 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:47:07.918 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:47:07.981 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:47:08.010 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:47:08.129 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:27 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:08.248 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout:Install 2 Packages 2024-04-26T04:47:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:08.250 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 892 k 2024-04-26T04:47:08.250 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 2.6 M 2024-04-26T04:47:08.250 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:47:08.756 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:47:08.756 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:47:08.756 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:47:08.756 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:47:08.756 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:47:08.757 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:47:08.840 INFO:teuthology.orchestra.run.smithi152.stdout:(1/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 1.6 MB/s | 812 kB 00:00 2024-04-26T04:47:09.024 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:09.025 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:09.148 INFO:teuthology.orchestra.run.smithi152.stdout:(2/2): fuse-2.9.9-15.el9.x86_64.rpm 98 kB/s | 80 kB 00:00 2024-04-26T04:47:09.149 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:09.150 INFO:teuthology.orchestra.run.smithi152.stdout:Total 992 kB/s | 892 kB 00:00 2024-04-26T04:47:09.161 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:47:09.184 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:47:09.184 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:47:09.261 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:47:09.261 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:47:09.262 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install ceph-fuse 2024-04-26T04:47:09.449 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:47:09.585 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:47:09.654 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:47:09.770 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:27 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:09.889 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:09.889 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:09.889 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repository Size 2024-04-26T04:47:09.889 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:09.889 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:47:09.889 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:47:09.889 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-04-26T04:47:09.890 INFO:teuthology.orchestra.run.smithi195.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:47:09.890 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:09.890 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:47:09.890 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:09.890 INFO:teuthology.orchestra.run.smithi195.stdout:Install 2 Packages 2024-04-26T04:47:09.890 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:09.891 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 892 k 2024-04-26T04:47:09.891 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 2.6 M 2024-04-26T04:47:09.891 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:47:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:47:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:47:10.364 INFO:teuthology.orchestra.run.smithi195.stdout:(1/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 2.4 MB/s | 812 kB 00:00 2024-04-26T04:47:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:47:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:10.557 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:47:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:10.558 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:10.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados-devel 2024-04-26T04:47:10.965 INFO:teuthology.orchestra.run.smithi195.stdout:(2/2): fuse-2.9.9-15.el9.x86_64.rpm 86 kB/s | 80 kB 00:00 2024-04-26T04:47:10.966 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:10.966 INFO:teuthology.orchestra.run.smithi195.stdout:Total 829 kB/s | 892 kB 00:01 2024-04-26T04:47:10.978 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:11.001 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:11.001 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:11.079 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:11.079 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:11.275 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:11.306 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:30 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:11.403 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:47:11.427 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:11.428 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-26T04:47:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:47:11.428 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:47:11.428 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:47:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:47:11.429 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 126 k 2024-04-26T04:47:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 448 k 2024-04-26T04:47:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:47:11.471 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:47:11.771 INFO:teuthology.orchestra.run.smithi152.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 370 kB/s | 126 kB 00:00 2024-04-26T04:47:11.771 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:11.772 INFO:teuthology.orchestra.run.smithi152.stdout:Total 368 kB/s | 126 kB 00:00 2024-04-26T04:47:11.772 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:47:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:47:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:47:11.840 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:47:11.840 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:47:11.990 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:47:12.070 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:47:12.070 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:47:12.128 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:12.324 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:47:12.325 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:12.325 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:12.325 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:47:12.325 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:12.325 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:12.529 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:12.576 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install librados-devel 2024-04-26T04:47:12.763 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:12.763 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:12.763 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:12.764 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:12.764 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:12.764 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:12.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs2 2024-04-26T04:47:13.094 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:31 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:13.215 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repo Size 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 126 k 2024-04-26T04:47:13.216 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 448 k 2024-04-26T04:47:13.217 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:47:13.494 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:32 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:13.545 INFO:teuthology.orchestra.run.smithi195.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 384 kB/s | 126 kB 00:00 2024-04-26T04:47:13.546 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:13.546 INFO:teuthology.orchestra.run.smithi195.stdout:Total 382 kB/s | 126 kB 00:00 2024-04-26T04:47:13.546 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:13.552 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:13.552 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:13.575 INFO:teuthology.orchestra.run.smithi152.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:13.614 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:13.615 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:13.617 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:13.617 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:47:13.617 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:13.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs-devel 2024-04-26T04:47:13.767 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:13.892 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:14.205 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:33 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:14.297 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:14.325 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:14.326 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:14.326 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-26T04:47:14.326 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:14.326 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 25 k 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 110 k 2024-04-26T04:47:14.327 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:47:14.509 INFO:teuthology.orchestra.run.smithi152.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 137 kB/s | 25 kB 00:00 2024-04-26T04:47:14.510 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:14.510 INFO:teuthology.orchestra.run.smithi152.stdout:Total 136 kB/s | 25 kB 00:00 2024-04-26T04:47:14.510 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:47:14.513 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:47:14.513 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:47:14.515 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:47:14.515 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:47:14.519 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:14.520 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:14.520 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:14.520 INFO:teuthology.orchestra.run.smithi195.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:14.520 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:14.520 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:14.603 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:47:14.712 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:14.738 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install libcephfs2 2024-04-26T04:47:15.064 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:15.251 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:33 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:15.280 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:15.281 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:15.281 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:15.281 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:15.281 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:15.281 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:15.331 INFO:teuthology.orchestra.run.smithi195.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:15.372 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:15.373 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:47:15.373 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:15.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados2 2024-04-26T04:47:15.442 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install libcephfs-devel 2024-04-26T04:47:15.951 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:34 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:15.953 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:33 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:16.033 INFO:teuthology.orchestra.run.smithi152.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:16.073 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:16.073 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:16.073 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repo Size 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 25 k 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 110 k 2024-04-26T04:47:16.074 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:47:16.075 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:16.075 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:47:16.075 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:16.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librbd1 2024-04-26T04:47:16.242 INFO:teuthology.orchestra.run.smithi195.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 149 kB/s | 25 kB 00:00 2024-04-26T04:47:16.242 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:16.242 INFO:teuthology.orchestra.run.smithi195.stdout:Total 148 kB/s | 25 kB 00:00 2024-04-26T04:47:16.243 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:16.246 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:16.246 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:16.247 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:16.248 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:16.329 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:16.429 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:16.645 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:16.723 INFO:teuthology.orchestra.run.smithi152.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:16.761 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:16.764 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:16.765 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:47:16.765 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:16.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rados 2024-04-26T04:47:16.963 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:16.964 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:16.964 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:16.964 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:16.964 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:16.964 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:17.131 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install librados2 2024-04-26T04:47:17.339 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:17.418 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:17.458 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:17.460 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:47:17.460 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:17.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rgw 2024-04-26T04:47:17.635 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:17.714 INFO:teuthology.orchestra.run.smithi195.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:17.754 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:17.755 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:47:17.755 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:17.816 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install librbd1 2024-04-26T04:47:18.040 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:18.119 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:18.159 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:18.160 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:47:18.160 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:18.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-cephfs 2024-04-26T04:47:18.321 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:18.399 INFO:teuthology.orchestra.run.smithi195.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:18.440 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:18.441 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:47:18.441 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:18.499 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install python3-rados 2024-04-26T04:47:18.742 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:18.823 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:18.864 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:18.865 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:47:18.865 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:18.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rbd 2024-04-26T04:47:19.010 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:19.089 INFO:teuthology.orchestra.run.smithi195.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:19.130 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:19.131 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:47:19.131 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:19.194 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install python3-rgw 2024-04-26T04:47:19.441 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:19.521 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:19.561 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:19.562 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-26T04:47:19.562 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:19.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-fuse 2024-04-26T04:47:19.701 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:19.780 INFO:teuthology.orchestra.run.smithi195.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:19.820 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:19.821 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:47:19.821 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:19.886 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install python3-cephfs 2024-04-26T04:47:20.157 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:20.277 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:20.278 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 82 k 2024-04-26T04:47:20.279 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 222 k 2024-04-26T04:47:20.279 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:47:20.393 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:20.472 INFO:teuthology.orchestra.run.smithi195.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:20.487 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 392 kB/s | 82 kB 00:00 2024-04-26T04:47:20.488 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:20.488 INFO:teuthology.orchestra.run.smithi152.stdout:Total 390 kB/s | 82 kB 00:00 2024-04-26T04:47:20.488 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:47:20.495 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:47:20.495 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:47:20.513 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:20.514 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:47:20.514 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:20.557 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:47:20.557 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:47:20.578 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install python3-rbd 2024-04-26T04:47:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:47:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:21.086 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:21.164 INFO:teuthology.orchestra.run.smithi195.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:21.204 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:21.205 INFO:teuthology.orchestra.run.smithi195.stdout:Nothing to do. 2024-04-26T04:47:21.205 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:21.265 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install rbd-fuse 2024-04-26T04:47:21.430 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:21.430 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:21.430 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:21.430 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:21.430 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:21.430 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:21.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-mirror 2024-04-26T04:47:21.775 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:21.894 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:21.895 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:47:21.896 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:21.896 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 82 k 2024-04-26T04:47:21.896 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 222 k 2024-04-26T04:47:21.896 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:47:22.079 INFO:teuthology.orchestra.run.smithi195.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 451 kB/s | 82 kB 00:00 2024-04-26T04:47:22.079 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:22.079 INFO:teuthology.orchestra.run.smithi195.stdout:Total 448 kB/s | 82 kB 00:00 2024-04-26T04:47:22.079 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:22.086 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:22.086 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:22.147 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:22.147 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:22.188 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:41 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:22.295 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:22.309 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:22.310 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:22.310 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-26T04:47:22.310 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:22.310 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:47:22.310 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.9 M 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 12 M 2024-04-26T04:47:22.311 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:47:22.390 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:22.721 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:22.891 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 5.1 MB/s | 2.9 MB 00:00 2024-04-26T04:47:22.892 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:22.892 INFO:teuthology.orchestra.run.smithi152.stdout:Total 5.0 MB/s | 2.9 MB 00:00 2024-04-26T04:47:22.892 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:47:22.902 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:47:22.902 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:47:22.982 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:47:22.982 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:47:22.989 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:22.989 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:22.990 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:22.990 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:22.990 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:22.990 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:23.245 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install rbd-mirror 2024-04-26T04:47:23.396 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:47:23.450 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:23.475 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:23.476 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:47:23.476 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:47:23.476 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:23.476 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:23.476 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:23.759 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:41 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:23.881 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout: Package Arch Version Repo Size 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:23.882 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 2.9 M 2024-04-26T04:47:23.883 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 12 M 2024-04-26T04:47:23.883 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:47:24.432 INFO:teuthology.orchestra.run.smithi195.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 5.4 MB/s | 2.9 MB 00:00 2024-04-26T04:47:24.432 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:24.432 INFO:teuthology.orchestra.run.smithi195.stdout:Total 5.3 MB/s | 2.9 MB 00:00 2024-04-26T04:47:24.433 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:24.442 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:24.442 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:24.487 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:24.487 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:24.487 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:24.487 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:24.487 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:24.487 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:24.521 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:24.521 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:24.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-nbd 2024-04-26T04:47:24.938 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:24.999 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:25.022 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:25.022 INFO:teuthology.orchestra.run.smithi195.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:47:25.022 INFO:teuthology.orchestra.run.smithi195.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:47:25.023 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:25.023 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:25.023 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:25.235 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:44 ago on Fri 26 Apr 2024 04:45:41 AM UTC. 2024-04-26T04:47:25.359 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-26T04:47:25.359 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 165 k 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 477 k 2024-04-26T04:47:25.360 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-26T04:47:25.588 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 728 kB/s | 165 kB 00:00 2024-04-26T04:47:25.588 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout:Total 722 kB/s | 165 kB 00:00 2024-04-26T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-26T04:47:25.596 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-26T04:47:25.596 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-26T04:47:25.659 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-26T04:47:25.660 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-26T04:47:25.809 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-26T04:47:25.908 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:25.979 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:25.979 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:25.980 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:25.980 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:25.980 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:25.980 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:26.200 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum -y install rbd-nbd 2024-04-26T04:47:26.299 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:26.561 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:26.561 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:26.561 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-26T04:47:26.562 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:26.562 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:47:26.562 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-26T04:47:26.717 INFO:teuthology.orchestra.run.smithi195.stdout:Last metadata expiration check: 0:01:44 ago on Fri 26 Apr 2024 04:45:42 AM UTC. 2024-04-26T04:47:26.797 DEBUG:teuthology.parallel:result is None 2024-04-26T04:47:26.838 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-04-26T04:47:26.838 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:26.838 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:Install 1 Package 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 165 k 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 477 k 2024-04-26T04:47:26.839 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-04-26T04:47:27.060 INFO:teuthology.orchestra.run.smithi195.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 748 kB/s | 165 kB 00:00 2024-04-26T04:47:27.060 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:27.061 INFO:teuthology.orchestra.run.smithi195.stdout:Total 744 kB/s | 165 kB 00:00 2024-04-26T04:47:27.061 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-04-26T04:47:27.068 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-04-26T04:47:27.068 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-04-26T04:47:27.131 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-04-26T04:47:27.132 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-04-26T04:47:27.276 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-04-26T04:47:27.372 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:27.785 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:28.013 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:28.014 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:28.014 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-04-26T04:47:28.014 INFO:teuthology.orchestra.run.smithi195.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:28.014 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:47:28.014 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-04-26T04:47:28.229 DEBUG:teuthology.parallel:result is None 2024-04-26T04:47:28.229 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:47:28.229 INFO:teuthology.packaging:ref: None 2024-04-26T04:47:28.229 INFO:teuthology.packaging:tag: None 2024-04-26T04:47:28.229 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:47:28.230 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:47:28.230 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:47:28.568 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:47:28.593 INFO:teuthology.orchestra.run.smithi152.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:28.594 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:28.594 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:47:28.595 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:47:28.596 INFO:teuthology.packaging:ref: None 2024-04-26T04:47:28.596 INFO:teuthology.packaging:tag: None 2024-04-26T04:47:28.596 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:47:28.596 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:47:28.596 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:47:28.829 DEBUG:teuthology.orchestra.run.smithi195:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:47:28.851 INFO:teuthology.orchestra.run.smithi195.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:28.852 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:28.852 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:47:28.854 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-26T04:47:28.854 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:47:28.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:47:28.884 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:47:28.885 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:47:28.916 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-26T04:47:28.916 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:47:28.916 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:47:28.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:47:29.019 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:47:29.019 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:47:29.043 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:47:29.108 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-26T04:47:29.108 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:47:29.108 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:47:29.135 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:47:29.202 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:47:29.202 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:47:29.225 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:47:29.287 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-26T04:47:29.288 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:47:29.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:47:29.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:47:29.378 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:47:29.378 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:47:29.402 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:47:29.466 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-26T04:47:29.570 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:47:29.570 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:47:29.571 INFO:tasks.cephadm:Cluster fsid is 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:47:29.571 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-26T04:47:29.571 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.152', 'mon.c': '[v2:172.21.15.152:3301,v1:172.21.15.152:6790]', 'mon.b': '172.21.15.195'} 2024-04-26T04:47:29.571 INFO:tasks.cephadm:First mon is mon.a on smithi152 2024-04-26T04:47:29.571 INFO:tasks.cephadm:First mgr is y 2024-04-26T04:47:29.571 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-26T04:47:29.572 DEBUG:teuthology.orchestra.run.smithi152:> sudo hostname $(hostname -s) 2024-04-26T04:47:29.598 DEBUG:teuthology.orchestra.run.smithi195:> sudo hostname $(hostname -s) 2024-04-26T04:47:29.622 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-26T04:47:29.623 DEBUG:teuthology.orchestra.run.smithi152:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:29.807 INFO:teuthology.orchestra.run.smithi152.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:47 /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:29.807 DEBUG:teuthology.orchestra.run.smithi195:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:30.316 INFO:teuthology.orchestra.run.smithi195.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:47 /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:30.317 DEBUG:teuthology.orchestra.run.smithi152:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:30.334 DEBUG:teuthology.orchestra.run.smithi195:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:30.363 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-26T04:47:30.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:47:30.376 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:47:32.059 INFO:teuthology.orchestra.run.smithi195.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:47:32.060 INFO:teuthology.orchestra.run.smithi152.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:49:05.977 INFO:teuthology.orchestra.run.smithi195.stdout:{ 2024-04-26T04:49:05.977 INFO:teuthology.orchestra.run.smithi195.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:49:05.977 INFO:teuthology.orchestra.run.smithi195.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:49:05.978 INFO:teuthology.orchestra.run.smithi195.stdout: "repo_digests": [ 2024-04-26T04:49:05.978 INFO:teuthology.orchestra.run.smithi195.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:49:05.978 INFO:teuthology.orchestra.run.smithi195.stdout: ] 2024-04-26T04:49:05.978 INFO:teuthology.orchestra.run.smithi195.stdout:} 2024-04-26T04:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-26T04:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "repo_digests": [ 2024-04-26T04:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-04-26T04:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-26T04:49:06.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo mkdir -p /etc/ceph 2024-04-26T04:49:06.801 DEBUG:teuthology.orchestra.run.smithi195:> sudo mkdir -p /etc/ceph 2024-04-26T04:49:06.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 777 /etc/ceph 2024-04-26T04:49:06.852 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 777 /etc/ceph 2024-04-26T04:49:06.892 INFO:tasks.cephadm:Writing seed config... 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-26T04:49:06.894 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-26T04:49:06.895 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:49:06.895 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-26T04:49:06.909 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 164326a4-0388-11ef-bc93-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-26T04:49:06.910 DEBUG:teuthology.orchestra.run.smithi152:mon.a> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service 2024-04-26T04:49:06.952 DEBUG:teuthology.orchestra.run.smithi152:mgr.y> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service 2024-04-26T04:49:06.994 INFO:tasks.cephadm:Bootstrapping... 2024-04-26T04:49:06.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 164326a4-0388-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.152 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:07.196 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:49:07.196 INFO:teuthology.orchestra.run.smithi152.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '164326a4-0388-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.152', '--skip-admin-label'] 2024-04-26T04:49:07.222 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:49:07.222 INFO:teuthology.orchestra.run.smithi152.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-26T04:49:07.223 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying podman|docker is present... 2024-04-26T04:49:07.246 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:49:07.246 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying lvm2 is present... 2024-04-26T04:49:07.246 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying time synchronization is in place... 2024-04-26T04:49:07.254 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:49:07.254 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:49:07.261 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:49:07.261 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-26T04:49:07.267 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout enabled 2024-04-26T04:49:07.274 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout active 2024-04-26T04:49:07.274 INFO:teuthology.orchestra.run.smithi152.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:49:07.274 INFO:teuthology.orchestra.run.smithi152.stdout:Repeating the final host check... 2024-04-26T04:49:07.297 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:49:07.297 INFO:teuthology.orchestra.run.smithi152.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-26T04:49:07.298 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl is present 2024-04-26T04:49:07.298 INFO:teuthology.orchestra.run.smithi152.stdout:lvcreate is present 2024-04-26T04:49:07.304 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:49:07.304 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:49:07.310 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:49:07.310 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-26T04:49:07.316 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout enabled 2024-04-26T04:49:07.324 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout active 2024-04-26T04:49:07.325 INFO:teuthology.orchestra.run.smithi152.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:49:07.325 INFO:teuthology.orchestra.run.smithi152.stdout:Host looks OK 2024-04-26T04:49:07.325 INFO:teuthology.orchestra.run.smithi152.stdout:Cluster fsid: 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:07.325 INFO:teuthology.orchestra.run.smithi152.stdout:Acquiring lock 139963410464976 on /run/cephadm/164326a4-0388-11ef-bc93-c7b262605968.lock 2024-04-26T04:49:07.325 INFO:teuthology.orchestra.run.smithi152.stdout:Lock 139963410464976 acquired on /run/cephadm/164326a4-0388-11ef-bc93-c7b262605968.lock 2024-04-26T04:49:07.326 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying IP 172.21.15.152 port 3300 ... 2024-04-26T04:49:07.326 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying IP 172.21.15.152 port 6789 ... 2024-04-26T04:49:07.326 INFO:teuthology.orchestra.run.smithi152.stdout:Base mon IP(s) is [172.21.15.152:3300, 172.21.15.152:6789], mon addrv is [v2:172.21.15.152:3300,v1:172.21.15.152:6789] 2024-04-26T04:49:07.327 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.152 metric 100 2024-04-26T04:49:07.328 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.152 metric 100 2024-04-26T04:49:07.330 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-26T04:49:07.331 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-26T04:49:07.331 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-26T04:49:07.334 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-26T04:49:07.334 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-26T04:49:07.334 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:49:07.334 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-26T04:49:07.334 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout inet6 fe80::1657:5650:b453:dded/64 scope link noprefixroute 2024-04-26T04:49:07.335 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:49:07.335 INFO:teuthology.orchestra.run.smithi152.stdout:Mon IP `172.21.15.152` is in CIDR network `172.21.0.0/20` 2024-04-26T04:49:07.335 INFO:teuthology.orchestra.run.smithi152.stdout:Mon IP `172.21.15.152` is in CIDR network `172.21.0.0/20` 2024-04-26T04:49:07.335 INFO:teuthology.orchestra.run.smithi152.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-26T04:49:07.336 INFO:teuthology.orchestra.run.smithi152.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-26T04:49:07.336 INFO:teuthology.orchestra.run.smithi152.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:49:07.957 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:49:07.958 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:49:07.958 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Getting image source signatures 2024-04-26T04:49:07.958 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Copying blob sha256:0e2af62137028b073c6ccdb8d4d7e0ccccc475ce5f6e225cb983e90813813045 2024-04-26T04:49:07.958 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Copying blob sha256:3d665f4c032644be9cae748af04bee69f2bfe96cdc3616dabe0ab17899633ab5 2024-04-26T04:49:07.958 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Copying config sha256:0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:49:07.958 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-26T04:49:08.954 INFO:teuthology.orchestra.run.smithi152.stdout:ceph: stdout ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:49:08.955 INFO:teuthology.orchestra.run.smithi152.stdout:Ceph version: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:49:08.955 INFO:teuthology.orchestra.run.smithi152.stdout:Extracting ceph user uid/gid from container image... 2024-04-26T04:49:09.945 INFO:teuthology.orchestra.run.smithi152.stdout:stat: stdout 167 167 2024-04-26T04:49:09.960 INFO:teuthology.orchestra.run.smithi152.stdout:Creating initial keys... 2024-04-26T04:49:10.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph-authtool: stdout AQBGMitmlaU/ExAAMnh4BFJiD1tKEq84NId5ug== 2024-04-26T04:49:11.741 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph-authtool: stdout AQBHMitmaMI7DxAAe2RhgqMsCXFHfKGxVXYJnw== 2024-04-26T04:49:12.830 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph-authtool: stdout AQBIMitmN7RxBxAAhQFLPegZURDr++xs2BWf5g== 2024-04-26T04:49:12.831 INFO:teuthology.orchestra.run.smithi152.stdout:Creating initial monmap... 2024-04-26T04:49:13.794 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:49:13.794 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-26T04:49:13.794 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:13.794 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:49:13.794 INFO:teuthology.orchestra.run.smithi152.stdout:monmaptool for a [v2:172.21.15.152:3300,v1:172.21.15.152:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:49:13.794 INFO:teuthology.orchestra.run.smithi152.stdout:setting min_mon_release = octopus 2024-04-26T04:49:13.794 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: set fsid to 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:13.795 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:49:13.795 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:49:13.795 INFO:teuthology.orchestra.run.smithi152.stdout:Creating mon... 2024-04-26T04:49:14.774 INFO:teuthology.orchestra.run.smithi152.stdout:create mon.a on 2024-04-26T04:49:14.973 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-26T04:49:15.112 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-26T04:49:15.257 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-164326a4-0388-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-164326a4-0388-11ef-bc93-c7b262605968.target. 2024-04-26T04:49:15.258 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-164326a4-0388-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-164326a4-0388-11ef-bc93-c7b262605968.target. 2024-04-26T04:49:15.420 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a 2024-04-26T04:49:15.420 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to reset failed state of unit ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-26T04:49:15.589 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-164326a4-0388-11ef-bc93-c7b262605968.target.wants/ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-164326a4-0388-11ef-bc93-c7b262605968@.service. 2024-04-26T04:49:15.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:15 smithi152 podman[27049]: 2024-04-26 04:49:15.833560441 +0000 UTC m=+0.116634948 container create c34351627c412308a5b76a2a96d750d8cc66eab3ec0d3490c87edadad45dd2a7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:16.091 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:49:16.091 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-26T04:49:16.098 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:49:16.098 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-26T04:49:16.098 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-26T04:49:16.098 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:49:16.099 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mon to start... 2024-04-26T04:49:16.099 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mon... 2024-04-26T04:49:16.297 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:16 smithi152 podman[27049]: 2024-04-26 04:49:16.010986858 +0000 UTC m=+0.294061366 container init c34351627c412308a5b76a2a96d750d8cc66eab3ec0d3490c87edadad45dd2a7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-c61472c) 2024-04-26T04:49:16.298 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:16 smithi152 podman[27049]: 2024-04-26 04:49:16.016250083 +0000 UTC m=+0.299324594 container start c34351627c412308a5b76a2a96d750d8cc66eab3ec0d3490c87edadad45dd2a7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T04:49:16.298 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:16 smithi152 ceph-mon[27087]: mkfs 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:16.298 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:16 smithi152 ceph-mon[27087]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:49:16.298 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:16 smithi152 bash[27049]: c34351627c412308a5b76a2a96d750d8cc66eab3ec0d3490c87edadad45dd2a7 2024-04-26T04:49:16.298 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:16 smithi152 systemd[1]: Started Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:49:17.209 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:17 smithi152 ceph-mon[27087]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:49:17.209 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:17 smithi152 ceph-mon[27087]: monmap e1: 1 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:49:17.209 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:17 smithi152 ceph-mon[27087]: fsmap 2024-04-26T04:49:17.209 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:17 smithi152 ceph-mon[27087]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:49:17.209 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:17 smithi152 ceph-mon[27087]: mgrmap e1: no daemons active 2024-04-26T04:49:17.209 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:17 smithi152 ceph-mon[27087]: from='client.? 172.21.15.152:0/1310344100' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T04:49:17.213 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout cluster: 2024-04-26T04:49:17.213 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout id: 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:17.213 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-26T04:49:17.213 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:17.213 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout services: 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.555283s) 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout data: 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout pgs: 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:mon is available 2024-04-26T04:49:17.214 INFO:teuthology.orchestra.run.smithi152.stdout:Assimilating anything we can from ceph.conf... 2024-04-26T04:49:18.343 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:18 smithi152 ceph-mon[27087]: from='client.? 172.21.15.152:0/3760567555' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:49:18.343 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:18 smithi152 ceph-mon[27087]: from='client.? 172.21.15.152:0/3760567555' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-26T04:49:18.346 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout fsid = 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.152:3300,v1:172.21.15.152:6789] 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:18.347 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:49:18.348 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:49:18.348 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:49:18.348 INFO:teuthology.orchestra.run.smithi152.stdout:Generating new minimal ceph.conf... 2024-04-26T04:49:19.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:19 smithi152 ceph-mon[27087]: from='client.? 172.21.15.152:0/276940467' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:19.393 INFO:teuthology.orchestra.run.smithi152.stdout:Restarting the monitor... 2024-04-26T04:49:19.771 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:19 smithi152 systemd[1]: Stopping Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:49:19.771 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:19 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a[27064]: 2024-04-26T04:49:19.637+0000 7f0b31655700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:49:19.771 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:19 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a[27064]: 2024-04-26T04:49:19.637+0000 7f0b31655700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-26T04:49:20.060 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:19 smithi152 podman[27406]: 2024-04-26 04:49:19.770449985 +0000 UTC m=+0.220734453 container died c34351627c412308a5b76a2a96d750d8cc66eab3ec0d3490c87edadad45dd2a7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-26T04:49:20.343 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 podman[27406]: 2024-04-26 04:49:20.126198068 +0000 UTC m=+0.576482538 container cleanup c34351627c412308a5b76a2a96d750d8cc66eab3ec0d3490c87edadad45dd2a7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-26T04:49:20.343 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 bash[27406]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:49:20.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 podman[27425]: 2024-04-26 04:49:20.343515708 +0000 UTC m=+0.576501828 container remove c34351627c412308a5b76a2a96d750d8cc66eab3ec0d3490c87edadad45dd2a7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T04:49:20.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:49:20.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 systemd[1]: Stopped Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:49:20.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 systemd[1]: Starting Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:49:20.913 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 podman[27523]: 2024-04-26 04:49:20.558976665 +0000 UTC m=+0.017931768 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:49:20.914 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 podman[27523]: 2024-04-26 04:49:20.686065917 +0000 UTC m=+0.145021037 container create 343ee858bd2cbcd665f81e8e9bba0279a42c802addc842d8de6d3d15b3b99b93 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS) 2024-04-26T04:49:20.914 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 podman[27523]: 2024-04-26 04:49:20.888201649 +0000 UTC m=+0.347156735 container init 343ee858bd2cbcd665f81e8e9bba0279a42c802addc842d8de6d3d15b3b99b93 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:20.914 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 podman[27523]: 2024-04-26 04:49:20.893471002 +0000 UTC m=+0.352426078 container start 343ee858bd2cbcd665f81e8e9bba0279a42c802addc842d8de6d3d15b3b99b93 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-26T04:49:20.914 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:49:20.914 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 7 2024-04-26T04:49:20.914 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: pidfile_write: ignore empty --pid-file 2024-04-26T04:49:21.017 INFO:teuthology.orchestra.run.smithi152.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: load: jerasure load: lrc 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: DB SUMMARY 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: DB Session ID: 8F9ML08SJCAPF4YHKK4B 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84258 ; 2024-04-26T04:49:21.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.env: 0x558c85ee2b00 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.fs: Posix File System 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.info_log: 0x558c87466ee0 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.statistics: (nil) 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.use_fsync: 0 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:49:21.230 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.db_log_dir: 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:49:21.231 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.write_buffer_manager: 0x558c881d6930 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:49:21.233 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.unordered_write: 0 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.row_cache: None 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.wal_filter: None 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:49:21.234 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:49:21.235 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_open_files: -1 2024-04-26T04:49:21.236 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Compression algorithms supported: 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kZSTD supported: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:49:21.237 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.merge_operator: 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_filter: None 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558c874b6f58) 2024-04-26T04:49:21.238 INFO:journalctl@ceph.mon.a.smithi152.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: index_type: 0 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: data_block_index_type: 0 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: index_shortening: 1 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: hash_index_allow_collision: 1 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: checksum: 1 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: no_block_cache: 0 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache: 0x558c874e9090 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache_options: 2024-04-26T04:49:21.239 INFO:journalctl@ceph.mon.a.smithi152.stdout: capacity : 536870912 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: num_shard_bits : 4 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: strict_capacity_limit : 0 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache_compressed: (nil) 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: persistent_cache: (nil) 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_size: 4096 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_size_deviation: 10 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_restart_interval: 16 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: index_block_restart_interval: 1 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: metadata_block_size: 4096 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: partition_filters: 0 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: use_delta_encoding: 1 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:49:21.240 INFO:journalctl@ceph.mon.a.smithi152.stdout: whole_key_filtering: 1 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout: verify_compression: 0 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout: format_version: 4 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout: enable_index_compression: 1 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_align: 0 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression: NoCompression 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.num_levels: 7 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:49:21.241 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:21.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:49:21.243 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:49:21.244 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:49:21.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.ttl: 2592000 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-26T04:49:21.246 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106960919506, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106960920222, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80690, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78951, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73159, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106960, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "4903732b-dbb9-4672-b8a2-8e997a85500a", "db_session_id": "8F9ML08SJCAPF4YHKK4B"}} 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106960920533, "job": 1, "event": "recovery_finished"} 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x558c8747d180 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: DB pointer 0x558c87552000 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** DB Stats ** 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:21.247 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: L0 2/0 80.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: Sum 2/0 80.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:21.248 INFO:journalctl@ceph.mon.a.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative compaction: 0.00 GB write, 26.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval compaction: 0.00 GB write, 26.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:21.249 INFO:journalctl@ceph.mon.a.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: L0 2/0 80.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: Sum 2/0 80.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:21.250 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative compaction: 0.00 GB write, 26.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: starting mon.a rank 0 at public addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] at bind addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???) e1 preinit fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???).mds e1 new map 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???).mds e1 print_map 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: e1 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:49:21.251 INFO:journalctl@ceph.mon.a.smithi152.stdout: legacy client fscid: -1 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout: No filesystems configured 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: monmap e1: 1 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: fsmap 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:20 smithi152 ceph-mon[27560]: mgrmap e1: no daemons active 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:21 smithi152 bash[27523]: 343ee858bd2cbcd665f81e8e9bba0279a42c802addc842d8de6d3d15b3b99b93 2024-04-26T04:49:21.252 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:21 smithi152 systemd[1]: Started Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:49:22.072 INFO:teuthology.orchestra.run.smithi152.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-26T04:49:22.072 INFO:teuthology.orchestra.run.smithi152.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:22.072 INFO:teuthology.orchestra.run.smithi152.stdout:Creating mgr... 2024-04-26T04:49:22.072 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying port 9283 ... 2024-04-26T04:49:22.240 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y 2024-04-26T04:49:22.240 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to reset failed state of unit ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-26T04:49:22.385 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-164326a4-0388-11ef-bc93-c7b262605968.target.wants/ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-164326a4-0388-11ef-bc93-c7b262605968@.service. 2024-04-26T04:49:22.743 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:22 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4152867137' entity='client.admin' 2024-04-26T04:49:22.743 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:22 smithi152 podman[27804]: 2024-04-26 04:49:22.645024966 +0000 UTC m=+0.126967976 container create ee2b14613007676b556bcc14697c5c730d0e15fd494e75494374666b9c2bb2d5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424) 2024-04-26T04:49:22.943 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:49:22.943 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-26T04:49:22.951 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:49:22.951 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-26T04:49:22.951 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-26T04:49:22.951 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:49:22.959 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:49:22.959 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-26T04:49:22.967 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:49:22.967 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-26T04:49:22.967 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-26T04:49:22.967 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-26T04:49:22.967 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr to start... 2024-04-26T04:49:22.967 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr... 2024-04-26T04:49:23.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:22 smithi152 podman[27804]: 2024-04-26 04:49:22.847072306 +0000 UTC m=+0.329015258 container init ee2b14613007676b556bcc14697c5c730d0e15fd494e75494374666b9c2bb2d5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T04:49:23.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:22 smithi152 podman[27804]: 2024-04-26 04:49:22.852706776 +0000 UTC m=+0.334649725 container start ee2b14613007676b556bcc14697c5c730d0e15fd494e75494374666b9c2bb2d5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c) 2024-04-26T04:49:23.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:22 smithi152 bash[27804]: ee2b14613007676b556bcc14697c5c730d0e15fd494e75494374666b9c2bb2d5 2024-04-26T04:49:23.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:22 smithi152 systemd[1]: Started Ceph mgr.y for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:49:23.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:22 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:22.996+0000 7f77d901e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:23.358 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:23 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:23.159+0000 7f77d901e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:23.358 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:23 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:23.268+0000 7f77d901e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:23 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4055311272' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:49:23.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:23 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:23.598+0000 7f77d901e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:23.936 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:23 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:23.682+0000 7f77d901e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:25.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:24.844+0000 7f77d901e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:25.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:25.008+0000 7f77d901e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:25.439 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:25.177+0000 7f77d901e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:25.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:25.438+0000 7f77d901e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:25.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:25.526+0000 7f77d901e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:25.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:25.618+0000 7f77d901e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:26.284 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:25.893+0000 7f77d901e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:26.293 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:26.293 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:49:26.293 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsid": "164326a4-0388-11ef-bc93-c7b262605968", 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:49:26.294 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:49:26.295 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:49:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:49:26.297 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:49:16.055227+0000", 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:49:26.298 INFO:teuthology.orchestra.run.smithi152.stdout:mgr not available, waiting (1/15)... 2024-04-26T04:49:26.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:26 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:26.354+0000 7f77d901e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:27.169 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:26 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:26.898+0000 7f77d901e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:27.170 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:26 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:26.991+0000 7f77d901e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:27.170 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:27.167+0000 7f77d901e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:27.722 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:27.459+0000 7f77d901e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:27.723 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:27.720+0000 7f77d901e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:49:28.090 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:27.957+0000 7f77d901e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:49:28.090 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:28.088+0000 7f77d901e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:49:28.379 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:28.179+0000 7f77d901e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:49:28.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:28.293+0000 7f77d901e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:28.879 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:28.384+0000 7f77d901e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:49:29.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:29.179+0000 7f77d901e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:29.368+0000 7f77d901e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: Activating manager daemon y 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: mgrmap e2: y(active, starting, since 0.00274074s) 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:49:29.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: Manager daemon y is now available 2024-04-26T04:49:29.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:49:29.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:49:29.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' 2024-04-26T04:49:29.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' 2024-04-26T04:49:29.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:29 smithi152 ceph-mon[27560]: from='mgr.14100 172.21.15.152:0/3674924486' entity='mgr.y' 2024-04-26T04:49:31.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:31 smithi152 ceph-mon[27560]: mgrmap e3: y(active, since 1.00736s) 2024-04-26T04:49:32.571 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:32 smithi152 ceph-mon[27560]: mgrmap e4: y(active, since 2s) 2024-04-26T04:49:32.571 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:32 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2780850066' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:49:33.108 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsid": "164326a4-0388-11ef-bc93-c7b262605968", 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:33.109 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:49:33.110 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:49:33.111 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:33.112 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:49:33.113 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:49:16.055227+0000", 2024-04-26T04:49:33.114 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:49:33.114 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:49:33.114 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:49:33.114 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:49:33.114 INFO:teuthology.orchestra.run.smithi152.stdout:mgr is available 2024-04-26T04:49:34.088 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:33 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2288128501' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout fsid = 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:34.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:49:34.360 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:49:34.360 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-26T04:49:34.360 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:49:34.360 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:49:34.360 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:49:34.360 INFO:teuthology.orchestra.run.smithi152.stdout:Enabling cephadm module... 2024-04-26T04:49:35.377 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1516813938' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-26T04:49:35.377 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setuser ceph since I am not root 2024-04-26T04:49:35.377 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setgroup ceph since I am not root 2024-04-26T04:49:35.377 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:35.274+0000 7f0c6aac41c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:35.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:35.437+0000 7f0c6aac41c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:35.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:35.536+0000 7f0c6aac41c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:35.937 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:35.849+0000 7f0c6aac41c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:36.379 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:35.936+0000 7f0c6aac41c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:36 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1516813938' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-26T04:49:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:36 smithi152 ceph-mon[27560]: mgrmap e5: y(active, since 5s) 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for the mgr to restart... 2024-04-26T04:49:36.996 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr epoch 5... 2024-04-26T04:49:37.245 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:37 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1974220514' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:49:37.245 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:37.119+0000 7f0c6aac41c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:37.541 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:37.292+0000 7f0c6aac41c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:37.541 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:37.468+0000 7f0c6aac41c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:37.836 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:37.743+0000 7f0c6aac41c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:37.837 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:37.834+0000 7f0c6aac41c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:37.932+0000 7f0c6aac41c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:38.497 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:38 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:38.214+0000 7f0c6aac41c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:38.879 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:38 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:38.678+0000 7f0c6aac41c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:39.471 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:39.203+0000 7f0c6aac41c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:39.472 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:39.295+0000 7f0c6aac41c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:39.472 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:39.469+0000 7f0c6aac41c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:40.015 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:39.757+0000 7f0c6aac41c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:40.015 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:40.014+0000 7f0c6aac41c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:49:40.379 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:40.250+0000 7f0c6aac41c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:49:40.673 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:40.380+0000 7f0c6aac41c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:49:40.674 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:40.471+0000 7f0c6aac41c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:49:40.674 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:40.585+0000 7f0c6aac41c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:40.674 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:40.672+0000 7f0c6aac41c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:49:41.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:41.465+0000 7f0c6aac41c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:49:41.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:49:41.654+0000 7f0c6aac41c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:49:41.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: Active manager daemon y restarted 2024-04-26T04:49:41.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: Activating manager daemon y 2024-04-26T04:49:41.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: osdmap e2: 0 total, 0 up, 0 in 2024-04-26T04:49:41.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: mgrmap e6: y(active, starting, since 0.00299467s) 2024-04-26T04:49:41.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: Manager daemon y is now available 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:49:41.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:41 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:49:42.980 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:42 smithi152 ceph-mon[27560]: Found migration_current of "None". Setting to last migration. 2024-04-26T04:49:42.980 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:42 smithi152 ceph-mon[27560]: mgrmap e7: y(active, since 1.00476s) 2024-04-26T04:49:43.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:49:43.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-26T04:49:43.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:49:43.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:49:43.583 INFO:teuthology.orchestra.run.smithi152.stdout:mgr epoch 5 is available 2024-04-26T04:49:43.583 INFO:teuthology.orchestra.run.smithi152.stdout:Setting orchestrator backend to cephadm... 2024-04-26T04:49:44.255 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:43 smithi152 ceph-mon[27560]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:49:44.255 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:43 smithi152 ceph-mon[27560]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:49:44.255 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:43 smithi152 ceph-mon[27560]: [26/Apr/2024:04:49:42] ENGINE Bus STARTING 2024-04-26T04:49:44.255 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:43 smithi152 ceph-mon[27560]: [26/Apr/2024:04:49:42] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:49:44.255 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:43 smithi152 ceph-mon[27560]: [26/Apr/2024:04:49:42] ENGINE Bus STARTED 2024-04-26T04:49:44.255 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:43 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:44.255 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:43 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:45.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:44 smithi152 ceph-mon[27560]: mgrmap e8: y(active, since 2s) 2024-04-26T04:49:45.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:44 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:45.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:44 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:46.148 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:45 smithi152 ceph-mon[27560]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:46.151 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-26T04:49:46.151 INFO:teuthology.orchestra.run.smithi152.stdout:Generating ssh key... 2024-04-26T04:49:47.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:46 smithi152 ceph-mon[27560]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: Generating public/private rsa key pair. 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: Your identification has been saved in /tmp/tmpcr2jc0iv/key. 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: Your public key has been saved in /tmp/tmpcr2jc0iv/key.pub. 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: The key fingerprint is: 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: SHA256:iRPx1oKcLEB7xdCDrd69ZYbVTvDQwL7eC3xaWF3qSkw ceph-164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: The key's randomart image is: 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: +---[RSA 3072]----+ 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | .o .*o ..o | 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | o.+== .+ . | 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | . o.*.+..= .| 2024-04-26T04:49:47.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | ... + oo +. o | 2024-04-26T04:49:47.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | . .o.So +E o | 2024-04-26T04:49:47.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | . ..o.==.. | 2024-04-26T04:49:47.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | *+.= . | 2024-04-26T04:49:47.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | . .*.. | 2024-04-26T04:49:47.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: | . o. | 2024-04-26T04:49:47.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: +----[SHA256]-----+ 2024-04-26T04:49:48.100 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-mon[27560]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:48.100 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-mon[27560]: Generating ssh key... 2024-04-26T04:49:48.100 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:48.100 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:47 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:48.997 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdQD6vtKNE6M+hTqrAc8J4K/LBCpk1k23Y/7ebJvocH/PMgRqCz/D05W9hV8NVlFam0HWWEJ6hsnJAxIo02b4H5H4rZM0THB5PU2Aj12VyGNvO9WIC8mzN/9jzwnnaJSVjhJ+2vD6zvdhcaNKSGm0JT2Ze+HsaavwTiDuhjnyeGtpMqfWKMXCIhVtAv6RjrWoHPJgYEY3MBticJAQESa4AUMeT3PVTIpVq3bo/rvusINR33YhROTeW1PJvKw2TJH5Ka4ZuJ/bt1o1sljqArpQ8sr1oUsSyOnndFIZZSYXdAxU1FC37bnMF59EAxSzQvbofHoSr64zjxuUze/wEmqJcKFfqlVzynwx8gfBnvU39+PFU1pQm9KVaV7cDqa+eYV34c/RrbQWwB7VSt4OCfkSvphFlTT26pvA9fJ6EkcZIz+qV3rxUAI8cj2mSjkQX5loi5bPjO8SWJmnVOBywwPY4b3yD5O5xWqC0zsjLhSXNA9RrfdDMGyTlPsREL/ZvY78= ceph-164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:48.997 INFO:teuthology.orchestra.run.smithi152.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-26T04:49:48.997 INFO:teuthology.orchestra.run.smithi152.stdout:Adding key to root@localhost authorized_keys... 2024-04-26T04:49:48.997 INFO:teuthology.orchestra.run.smithi152.stdout:Adding host smithi152... 2024-04-26T04:49:49.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:49 smithi152 ceph-mon[27560]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:50.505 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:50 smithi152 ceph-mon[27560]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi152", "addr": "172.21.15.152", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:51.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:51 smithi152 ceph-mon[27560]: Deploying cephadm binary to smithi152 2024-04-26T04:49:52.078 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Added host 'smithi152' with addr '172.21.15.152' 2024-04-26T04:49:52.079 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying unmanaged mon service... 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:52 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:52 smithi152 ceph-mon[27560]: Added host smithi152 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:52 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:53.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-26T04:49:53.587 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying unmanaged mgr service... 2024-04-26T04:49:53.870 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:53 smithi152 ceph-mon[27560]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:53.870 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:53 smithi152 ceph-mon[27560]: Saving service mon spec with placement count:5 2024-04-26T04:49:53.870 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:53 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:55.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-26T04:49:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:55 smithi152 ceph-mon[27560]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:55 smithi152 ceph-mon[27560]: Saving service mgr spec with placement count:2 2024-04-26T04:49:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:55 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:56.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:56 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:56.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:56 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:49:56.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:56 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2420310095' entity='client.admin' 2024-04-26T04:49:58.550 INFO:teuthology.orchestra.run.smithi152.stdout:Enabling the dashboard module... 2024-04-26T04:49:59.318 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:49:58 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4249329498' entity='client.admin' 2024-04-26T04:50:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:50:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3519893518' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-26T04:50:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-mon[27560]: from='mgr.14116 172.21.15.152:0/204200348' entity='mgr.y' 2024-04-26T04:50:00.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setuser ceph since I am not root 2024-04-26T04:50:00.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setgroup ceph since I am not root 2024-04-26T04:50:00.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:00.260+0000 7fcc2fa7e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:50:00.747 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:00.426+0000 7fcc2fa7e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:50:00.747 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:00.525+0000 7fcc2fa7e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:50:01.096 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:00.838+0000 7fcc2fa7e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:50:01.097 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:00 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:00.920+0000 7fcc2fa7e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:50:01.367 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:01 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3519893518' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-26T04:50:01.368 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:01 smithi152 ceph-mon[27560]: mgrmap e9: y(active, since 18s) 2024-04-26T04:50:02.262 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:02.086+0000 7fcc2fa7e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:50:02.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:02 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1062678720' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:50:02.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:02.260+0000 7fcc2fa7e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:50:02.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:02.432+0000 7fcc2fa7e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:50:03.002 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:02.701+0000 7fcc2fa7e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:50:03.002 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:02.792+0000 7fcc2fa7e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:50:03.002 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:02.886+0000 7fcc2fa7e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:50:03.006 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:50:03.006 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-26T04:50:03.006 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:50:03.006 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:50:03.006 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:50:03.006 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:50:03.006 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for the mgr to restart... 2024-04-26T04:50:03.007 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr epoch 9... 2024-04-26T04:50:03.256 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:03.171+0000 7fcc2fa7e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:50:04.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:03.654+0000 7fcc2fa7e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:50:04.455 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:04.184+0000 7fcc2fa7e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:50:04.455 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:04.277+0000 7fcc2fa7e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:50:04.455 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:04.454+0000 7fcc2fa7e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:50:05.021 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:04.744+0000 7fcc2fa7e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:50:05.021 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:05 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:05.020+0000 7fcc2fa7e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:50:05.292 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:05 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:05.269+0000 7fcc2fa7e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:50:05.628 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:05 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:05.410+0000 7fcc2fa7e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:50:05.628 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:05 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:05.508+0000 7fcc2fa7e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:50:05.628 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:05 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:05.627+0000 7fcc2fa7e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:50:05.879 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:05 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:05.721+0000 7fcc2fa7e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:50:06.778 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:06.527+0000 7fcc2fa7e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:50:06.778 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:50:06.720+0000 7fcc2fa7e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:50:06.778 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: Active manager daemon y restarted 2024-04-26T04:50:06.778 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: Activating manager daemon y 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: osdmap e3: 0 total, 0 up, 0 in 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: mgrmap e10: y(active, starting, since 0.00574053s) 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: Manager daemon y is now available 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:50:06.779 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:50:08.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:08 smithi152 ceph-mon[27560]: [26/Apr/2024:04:50:06] ENGINE Bus STARTING 2024-04-26T04:50:08.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:08 smithi152 ceph-mon[27560]: [26/Apr/2024:04:50:07] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:50:08.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:08 smithi152 ceph-mon[27560]: [26/Apr/2024:04:50:07] ENGINE Bus STARTED 2024-04-26T04:50:08.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:08 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:08.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:08 smithi152 ceph-mon[27560]: mgrmap e11: y(active, since 1.00521s) 2024-04-26T04:50:08.365 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:50:08.365 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-26T04:50:08.365 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:50:08.365 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:50:08.366 INFO:teuthology.orchestra.run.smithi152.stdout:mgr epoch 9 is available 2024-04-26T04:50:08.366 INFO:teuthology.orchestra.run.smithi152.stdout:Generating a dashboard self-signed certificate... 2024-04-26T04:50:09.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:09 smithi152 ceph-mon[27560]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:50:09.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:09 smithi152 ceph-mon[27560]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:50:09.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:09 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:09.663 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-26T04:50:09.663 INFO:teuthology.orchestra.run.smithi152.stdout:Creating initial admin user... 2024-04-26T04:50:10.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:10 smithi152 ceph-mon[27560]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:10 smithi152 ceph-mon[27560]: mgrmap e12: y(active, since 2s) 2024-04-26T04:50:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:11.346 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:11 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:11.346 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:11 smithi152 ceph-mon[27560]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:11.346 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:11 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:12.144 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$MBp7w3ePNXp3B4x3D556xeDP3PgQ/sAt7BlY5U49uqFm30dasRyvC", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714107010, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-26T04:50:12.144 INFO:teuthology.orchestra.run.smithi152.stdout:Fetching dashboard port number... 2024-04-26T04:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 8443 2024-04-26T04:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-26T04:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-26T04:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-26T04:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-26T04:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout:Ceph Dashboard is now available at: 2024-04-26T04:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: URL: https://smithi152:8443/ 2024-04-26T04:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: User: admin 2024-04-26T04:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: Password: 0cy3mg4ty6 2024-04-26T04:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout:Saving cluster configuration to /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config directory 2024-04-26T04:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout:Enabling autotune for osd_memory_target 2024-04-26T04:50:14.093 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:13 smithi152 ceph-mon[27560]: mgrmap e13: y(active, since 6s) 2024-04-26T04:50:14.093 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:13 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2639379804' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout:Or, if you are only running a single cluster on this host: 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.106 INFO:teuthology.orchestra.run.smithi152.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout: ceph telemetry on 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout:For more information see: 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:16.107 INFO:teuthology.orchestra.run.smithi152.stdout:Bootstrap complete. 2024-04-26T04:50:16.140 INFO:tasks.cephadm:Fetching config... 2024-04-26T04:50:16.141 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:50:16.141 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-26T04:50:16.156 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-26T04:50:16.156 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:50:16.156 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-26T04:50:16.210 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-26T04:50:16.211 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:50:16.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-26T04:50:16.275 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-26T04:50:16.276 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:50:16.276 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-26T04:50:16.330 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-26T04:50:16.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdQD6vtKNE6M+hTqrAc8J4K/LBCpk1k23Y/7ebJvocH/PMgRqCz/D05W9hV8NVlFam0HWWEJ6hsnJAxIo02b4H5H4rZM0THB5PU2Aj12VyGNvO9WIC8mzN/9jzwnnaJSVjhJ+2vD6zvdhcaNKSGm0JT2Ze+HsaavwTiDuhjnyeGtpMqfWKMXCIhVtAv6RjrWoHPJgYEY3MBticJAQESa4AUMeT3PVTIpVq3bo/rvusINR33YhROTeW1PJvKw2TJH5Ka4ZuJ/bt1o1sljqArpQ8sr1oUsSyOnndFIZZSYXdAxU1FC37bnMF59EAxSzQvbofHoSr64zjxuUze/wEmqJcKFfqlVzynwx8gfBnvU39+PFU1pQm9KVaV7cDqa+eYV34c/RrbQWwB7VSt4OCfkSvphFlTT26pvA9fJ6EkcZIz+qV3rxUAI8cj2mSjkQX5loi5bPjO8SWJmnVOBywwPY4b3yD5O5xWqC0zsjLhSXNA9RrfdDMGyTlPsREL/ZvY78= ceph-164326a4-0388-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:50:16.402 INFO:teuthology.orchestra.run.smithi152.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdQD6vtKNE6M+hTqrAc8J4K/LBCpk1k23Y/7ebJvocH/PMgRqCz/D05W9hV8NVlFam0HWWEJ6hsnJAxIo02b4H5H4rZM0THB5PU2Aj12VyGNvO9WIC8mzN/9jzwnnaJSVjhJ+2vD6zvdhcaNKSGm0JT2Ze+HsaavwTiDuhjnyeGtpMqfWKMXCIhVtAv6RjrWoHPJgYEY3MBticJAQESa4AUMeT3PVTIpVq3bo/rvusINR33YhROTeW1PJvKw2TJH5Ka4ZuJ/bt1o1sljqArpQ8sr1oUsSyOnndFIZZSYXdAxU1FC37bnMF59EAxSzQvbofHoSr64zjxuUze/wEmqJcKFfqlVzynwx8gfBnvU39+PFU1pQm9KVaV7cDqa+eYV34c/RrbQWwB7VSt4OCfkSvphFlTT26pvA9fJ6EkcZIz+qV3rxUAI8cj2mSjkQX5loi5bPjO8SWJmnVOBywwPY4b3yD5O5xWqC0zsjLhSXNA9RrfdDMGyTlPsREL/ZvY78= ceph-164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:16.412 DEBUG:teuthology.orchestra.run.smithi195:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdQD6vtKNE6M+hTqrAc8J4K/LBCpk1k23Y/7ebJvocH/PMgRqCz/D05W9hV8NVlFam0HWWEJ6hsnJAxIo02b4H5H4rZM0THB5PU2Aj12VyGNvO9WIC8mzN/9jzwnnaJSVjhJ+2vD6zvdhcaNKSGm0JT2Ze+HsaavwTiDuhjnyeGtpMqfWKMXCIhVtAv6RjrWoHPJgYEY3MBticJAQESa4AUMeT3PVTIpVq3bo/rvusINR33YhROTeW1PJvKw2TJH5Ka4ZuJ/bt1o1sljqArpQ8sr1oUsSyOnndFIZZSYXdAxU1FC37bnMF59EAxSzQvbofHoSr64zjxuUze/wEmqJcKFfqlVzynwx8gfBnvU39+PFU1pQm9KVaV7cDqa+eYV34c/RrbQWwB7VSt4OCfkSvphFlTT26pvA9fJ6EkcZIz+qV3rxUAI8cj2mSjkQX5loi5bPjO8SWJmnVOBywwPY4b3yD5O5xWqC0zsjLhSXNA9RrfdDMGyTlPsREL/ZvY78= ceph-164326a4-0388-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:50:16.453 INFO:teuthology.orchestra.run.smithi195.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdQD6vtKNE6M+hTqrAc8J4K/LBCpk1k23Y/7ebJvocH/PMgRqCz/D05W9hV8NVlFam0HWWEJ6hsnJAxIo02b4H5H4rZM0THB5PU2Aj12VyGNvO9WIC8mzN/9jzwnnaJSVjhJ+2vD6zvdhcaNKSGm0JT2Ze+HsaavwTiDuhjnyeGtpMqfWKMXCIhVtAv6RjrWoHPJgYEY3MBticJAQESa4AUMeT3PVTIpVq3bo/rvusINR33YhROTeW1PJvKw2TJH5Ka4ZuJ/bt1o1sljqArpQ8sr1oUsSyOnndFIZZSYXdAxU1FC37bnMF59EAxSzQvbofHoSr64zjxuUze/wEmqJcKFfqlVzynwx8gfBnvU39+PFU1pQm9KVaV7cDqa+eYV34c/RrbQWwB7VSt4OCfkSvphFlTT26pvA9fJ6EkcZIz+qV3rxUAI8cj2mSjkQX5loi5bPjO8SWJmnVOBywwPY4b3yD5O5xWqC0zsjLhSXNA9RrfdDMGyTlPsREL/ZvY78= ceph-164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:16.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-26T04:50:16.682 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:16 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3202159021' entity='client.admin' 2024-04-26T04:50:16.755 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:20.077 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-26T04:50:20.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-26T04:50:20.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:20 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3409353295' entity='client.admin' 2024-04-26T04:50:20.755 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:21.864 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:21.864 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:21.864 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi152", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:50:21.864 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:21.864 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:21.865 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:23.399 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi195 2024-04-26T04:50:23.400 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:50:23.400 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.conf 2024-04-26T04:50:23.418 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:50:23.418 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:23.475 INFO:tasks.cephadm:Adding host smithi195 to orchestrator... 2024-04-26T04:50:23.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch host add smithi195 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:23.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:23.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:23.724 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:24.823 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:24 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:26.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:26 smithi152 ceph-mon[27560]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi195", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:27.423 INFO:teuthology.orchestra.run.smithi152.stdout:Added host 'smithi195' with addr '172.21.15.195' 2024-04-26T04:50:28.004 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:27 smithi152 ceph-mon[27560]: Deploying cephadm binary to smithi195 2024-04-26T04:50:28.004 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:27 smithi152 ceph-mon[27560]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:28.004 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:28.004 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:28.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-26T04:50:28.274 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:29.036 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:28 smithi152 ceph-mon[27560]: Added host smithi195 2024-04-26T04:50:29.036 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:28 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:29.036 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:28 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:30.073 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:50:30.073 INFO:teuthology.orchestra.run.smithi152.stdout:[{"addr": "172.21.15.152", "hostname": "smithi152", "labels": [], "status": ""}, {"addr": "172.21.15.195", "hostname": "smithi195", "labels": [], "status": ""}] 2024-04-26T04:50:30.073 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:29 smithi152 ceph-mon[27560]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:30.073 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:30.671 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-26T04:50:30.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-26T04:50:30.917 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:31.040 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:30 smithi152 ceph-mon[27560]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:50:32.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:31 smithi152 ceph-mon[27560]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:32.889 INFO:teuthology.orchestra.run.smithi152.stderr:adjusted tunables profile to default 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi195", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4150964996' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T04:50:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:33.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:33.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:32 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:33.490 INFO:tasks.cephadm:Adding mon.a on smithi152 2024-04-26T04:50:33.490 INFO:tasks.cephadm:Adding mon.c on smithi152 2024-04-26T04:50:33.490 INFO:tasks.cephadm:Adding mon.b on smithi195 2024-04-26T04:50:33.490 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi152:172.21.15.152=a;smithi152:[v2:172.21.15.152:3301,v1:172.21.15.152:6790]=c;smithi195:172.21.15.195=b' 2024-04-26T04:50:33.734 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:34.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:33 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:33 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:33 smithi152 ceph-mon[27560]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:33 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4150964996' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:50:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:33 smithi152 ceph-mon[27560]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:35.373 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mon update... 2024-04-26T04:50:35.976 DEBUG:teuthology.orchestra.run.smithi152:mon.c> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.c.service 2024-04-26T04:50:35.979 DEBUG:teuthology.orchestra.run.smithi195:mon.b> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.b.service 2024-04-26T04:50:35.981 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:50:35.981 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:36.183 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:35 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:37.056 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:37.368 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:36 smithi152 ceph-mon[27560]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi152:172.21.15.152=a;smithi152:[v2:172.21.15.152:3301,v1:172.21.15.152:6790]=c;smithi195:172.21.15.195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:37.368 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:36 smithi152 ceph-mon[27560]: Saving service mon spec with placement smithi152:172.21.15.152=a;smithi152:[v2:172.21.15.152:3301,v1:172.21.15.152:6790]=c;smithi195:172.21.15.195=b;count:3 2024-04-26T04:50:37.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:36 smithi152 ceph-mon[27560]: Deploying daemon mon.b on smithi195 2024-04-26T04:50:38.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:37 smithi152 ceph-mon[27560]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:40.005 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:50:40.005 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","modified":"2024-04-26T04:49:13.193487Z","created":"2024-04-26T04:49:13.193487Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T04:50:40.007 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-04-26T04:50:40.135 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:50:40.135 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.db_log_dir: 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:50:40.136 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:50:40.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:39 smithi152 ceph-mon[27560]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:50:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:50:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.write_buffer_manager: 0x557a9c334930 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.unordered_write: 0 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:50:40.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.row_cache: None 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.wal_filter: None 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:50:40.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:50:40.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_open_files: -1 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:50:40.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Compression algorithms supported: 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kZSTD supported: 0 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:50:40.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.merge_operator: 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_filter: None 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557a9b614f58) 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:50:40.538 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_type: 0 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_block_index_type: 0 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_shortening: 1 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: hash_index_allow_collision: 1 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: checksum: 1 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: no_block_cache: 0 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache: 0x557a9b647090 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_options: 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: capacity : 536870912 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: num_shard_bits : 4 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: strict_capacity_limit : 0 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:50:40.539 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_compressed: (nil) 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: persistent_cache: (nil) 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_size: 4096 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_size_deviation: 10 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_restart_interval: 16 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_block_restart_interval: 1 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: metadata_block_size: 4096 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: partition_filters: 0 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: use_delta_encoding: 1 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: whole_key_filtering: 1 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: verify_compression: 0 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: format_version: 4 2024-04-26T04:50:40.540 INFO:journalctl@ceph.mon.b.smithi195.stdout: enable_index_compression: 1 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_align: 0 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression: NoCompression 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.num_levels: 7 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:50:40.541 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:50:40.542 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:50:40.543 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:50:40.544 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:50:40.545 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.ttl: 2592000 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107040136241, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:50:40.546 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107040136554, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714107040, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f103e535-089e-428d-a0bd-5476bbbc0df7", "db_session_id": "ZZBS65T4LPLQ7P9GXUF8"}} 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107040136874, "job": 1, "event": "recovery_finished"} 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x557a9b5db180 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: DB pointer 0x557a9b6b0000 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** DB Stats ** 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:50:40.547 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:40.548 INFO:journalctl@ceph.mon.b.smithi195.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:40.549 INFO:journalctl@ceph.mon.b.smithi195.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:50:40.550 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: using public_addr v2:172.21.15.195:0/0 -> [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: starting mon.b rank -1 at public addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] at bind addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(???) e0 preinit fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).mds e1 new map 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: e1 2024-04-26T04:50:40.551 INFO:journalctl@ceph.mon.b.smithi195.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout: legacy client fscid: -1 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout: No filesystems configured 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:50:40.552 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/4150964996' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi152:172.21.15.152=a;smithi152:[v2:172.21.15.152:3301,v1:172.21.15.152:6790]=c;smithi195:172.21.15.195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: Saving service mon spec with placement smithi152:172.21.15.152=a;smithi152:[v2:172.21.15.152:3301,v1:172.21.15.152:6790]=c;smithi195:172.21.15.195=b;count:3 2024-04-26T04:50:40.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: Deploying daemon mon.b on smithi195 2024-04-26T04:50:40.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:40.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:40.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 ceph-mon[31427]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:50:40.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 bash[31375]: bf477f6a2b3d1e874af67aba1c9c4c6a7d68894a4fb86d2be3e57184c76a8dcf 2024-04-26T04:50:40.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:40 smithi195 systemd[1]: Started Ceph mon.b for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:50:41.833 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:50:41.834 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:50:42.089 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:50:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: mon.a calling monitor election 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: mon.b calling monitor election 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: monmap e2: 2 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: fsmap 2024-04-26T04:50:45.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:45.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: mgrmap e13: y(active, since 38s) 2024-04-26T04:50:45.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: overall HEALTH_OK 2024-04-26T04:50:45.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:45.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: mon.a calling monitor election 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: mon.b calling monitor election 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: monmap e2: 2 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: fsmap 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: mgrmap e13: y(active, since 38s) 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: overall HEALTH_OK 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:45.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:46.337 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:46 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:46.337 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:46 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:46.337 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:46 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:46.337 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:46 smithi152 ceph-mon[27560]: Deploying daemon mon.c on smithi152 2024-04-26T04:50:46.337 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:46 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:46.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:46 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:46 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:46 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:46 smithi195 ceph-mon[31427]: Deploying daemon mon.c on smithi152 2024-04-26T04:50:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:46 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:46.844 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:50:46.844 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":2,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","modified":"2024-04-26T04:50:40.156463Z","created":"2024-04-26T04:49:13.193487Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:3300","nonce":0},{"type":"v1","addr":"172.21.15.195:6789","nonce":0}]},"addr":"172.21.15.195:6789/0","public_addr":"172.21.15.195:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T04:50:46.846 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 2 2024-04-26T04:50:47.411 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:47 smithi195 ceph-mon[31427]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:47.411 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:47 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/2593844905' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:47 smithi152 ceph-mon[27560]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:47 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/2593844905' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:48.347 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 podman[39285]: 2024-04-26 04:50:48.238323765 +0000 UTC m=+0.121981932 container create f907cd5ba87f7c9bb45ae6a9ed85044eb5bd52548e227e770c27314e8d5243da (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T04:50:48.439 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:50:48.439 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:50:48.694 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:50:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 podman[39285]: 2024-04-26 04:50:48.448978845 +0000 UTC m=+0.332637010 container init f907cd5ba87f7c9bb45ae6a9ed85044eb5bd52548e227e770c27314e8d5243da (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c) 2024-04-26T04:50:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 podman[39285]: 2024-04-26 04:50:48.454221964 +0000 UTC m=+0.337880128 container start f907cd5ba87f7c9bb45ae6a9ed85044eb5bd52548e227e770c27314e8d5243da (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-26T04:50:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:50:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 7 2024-04-26T04:50:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pidfile_write: ignore empty --pid-file 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: load: jerasure load: lrc 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: DB SUMMARY 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: DB Session ID: A4VS8OZ9B9NY918RPA87 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:50:48.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.env: 0x562e8ce68b00 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.fs: Posix File System 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.info_log: 0x562e8dd56ee0 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.statistics: (nil) 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.use_fsync: 0 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:50:48.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.db_log_dir: 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:50:48.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.write_buffer_manager: 0x562e8eac6960 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:50:48.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.unordered_write: 0 2024-04-26T04:50:48.885 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:50:48.885 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:50:48.885 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:50:48.885 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:50:48.885 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.row_cache: None 2024-04-26T04:50:48.885 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.wal_filter: None 2024-04-26T04:50:48.885 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:50:48.886 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:50:48.886 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:50:48.887 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_open_files: -1 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:50:48.888 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Compression algorithms supported: 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kZSTD supported: 0 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-26T04:50:48.889 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.merge_operator: 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_filter: None 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562e8dda6f58) 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:50:48.890 INFO:journalctl@ceph.mon.c.smithi152.stdout: index_type: 0 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: data_block_index_type: 0 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: index_shortening: 1 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: hash_index_allow_collision: 1 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: checksum: 1 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: no_block_cache: 0 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache: 0x562e8ddd9090 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache_options: 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: capacity : 536870912 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: num_shard_bits : 4 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: strict_capacity_limit : 0 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:50:48.891 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache_compressed: (nil) 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: persistent_cache: (nil) 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_size: 4096 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_size_deviation: 10 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_restart_interval: 16 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: index_block_restart_interval: 1 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: metadata_block_size: 4096 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: partition_filters: 0 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: use_delta_encoding: 1 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: whole_key_filtering: 1 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: verify_compression: 0 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: format_version: 4 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: enable_index_compression: 1 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_align: 0 2024-04-26T04:50:48.892 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression: NoCompression 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.num_levels: 7 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:50:48.893 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:50:48.894 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:50:48.895 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:50:48.896 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.ttl: 2592000 2024-04-26T04:50:48.897 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107048480966, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:50:48.898 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107048481292, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714107048, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "91721876-f8ac-4a2b-bf45-added091b872", "db_session_id": "A4VS8OZ9B9NY918RPA87"}} 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107048481635, "job": 1, "event": "recovery_finished"} 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x562e8dd6d180 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: DB pointer 0x562e8de42000 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** DB Stats ** 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:50:48.899 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:48.900 INFO:journalctl@ceph.mon.c.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:48.901 INFO:journalctl@ceph.mon.c.smithi152.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:50:48.902 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: using public_addrv [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: starting mon.c rank -1 at public addrs [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] at bind addrs [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(???) e0 preinit fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).mds e1 new map 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: e1 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:50:48.903 INFO:journalctl@ceph.mon.c.smithi152.stdout: legacy client fscid: -1 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout: No filesystems configured 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:50:48.904 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/4150964996' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi152:172.21.15.152=a;smithi152:[v2:172.21.15.152:3301,v1:172.21.15.152:6790]=c;smithi195:172.21.15.195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:48.905 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: Saving service mon spec with placement smithi152:172.21.15.152=a;smithi152:[v2:172.21.15.152:3301,v1:172.21.15.152:6790]=c;smithi195:172.21.15.195=b;count:3 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: Deploying daemon mon.b on smithi195 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.a calling monitor election 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.b calling monitor election 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:48.906 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: monmap e2: 2 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: fsmap 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mgrmap e13: y(active, since 38s) 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: overall HEALTH_OK 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: Deploying daemon mon.c on smithi152 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:48.907 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/2593844905' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:48.908 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 ceph-mon[39324]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:50:48.908 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 bash[39285]: f907cd5ba87f7c9bb45ae6a9ed85044eb5bd52548e227e770c27314e8d5243da 2024-04-26T04:50:48.908 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:48 smithi152 systemd[1]: Started Ceph mon.c for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: mon.a calling monitor election 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: mon.b calling monitor election 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: mon.c calling monitor election 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: monmap e3: 3 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: fsmap 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: mgrmap e13: y(active, since 46s) 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: overall HEALTH_OK 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:53.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:53.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:53.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:53.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: mon.a calling monitor election 2024-04-26T04:50:53.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: mon.b calling monitor election 2024-04-26T04:50:53.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:53.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: mon.c calling monitor election 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: monmap e3: 3 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: fsmap 2024-04-26T04:50:53.883 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:53.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: mgrmap e13: y(active, since 46s) 2024-04-26T04:50:53.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: overall HEALTH_OK 2024-04-26T04:50:53.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:53.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:53.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:53.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:54.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:54.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: mon.a calling monitor election 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: mon.b calling monitor election 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: mon.c calling monitor election 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:50:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: monmap e3: 3 mons at {a=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: fsmap 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: mgrmap e13: y(active, since 46s) 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: overall HEALTH_OK 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:54.123 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-04-26T04:50:54.124 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":3,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","modified":"2024-04-26T04:50:48.501641Z","created":"2024-04-26T04:49:13.193487Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:3300","nonce":0},{"type":"v1","addr":"172.21.15.195:6789","nonce":0}]},"addr":"172.21.15.195:6789/0","public_addr":"172.21.15.195:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3301","nonce":0},{"type":"v1","addr":"172.21.15.152:6790","nonce":0}]},"addr":"172.21.15.152:6790/0","public_addr":"172.21.15.152:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-26T04:50:54.126 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 3 2024-04-26T04:50:54.866 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:54 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:54.866 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:54 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/3587755638' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:54.866 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:54 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:54 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:54.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:54 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/3587755638' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:54 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:54 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:54.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:54 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/3587755638' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:54.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:54 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:54.892 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-26T04:50:54.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-26T04:50:55.140 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[39324]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[27560]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:56.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:55 smithi195 ceph-mon[31427]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:50:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:50:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: Reconfiguring daemon mon.a on smithi152 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: Reconfiguring daemon mon.a on smithi152 2024-04-26T04:50:57.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:57.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: Reconfiguring daemon mon.a on smithi152 2024-04-26T04:50:57.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:57.626 INFO:teuthology.orchestra.run.smithi152.stdout:# minimal ceph.conf for 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:57.626 INFO:teuthology.orchestra.run.smithi152.stdout:[global] 2024-04-26T04:50:57.626 INFO:teuthology.orchestra.run.smithi152.stdout: fsid = 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:57.626 INFO:teuthology.orchestra.run.smithi152.stdout: mon_host = [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] 2024-04-26T04:50:57.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:57 smithi152 ceph-mon[27560]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:57.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:57 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/86189948' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:57.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:57 smithi152 ceph-mon[39324]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:57.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:57 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/86189948' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:58.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:57 smithi195 ceph-mon[31427]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:58.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:57 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/86189948' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:58.533 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-26T04:50:58.533 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:50:58.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:50:58.559 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:50:58.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:58.623 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:50:58.623 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:50:58.654 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:50:58.655 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:58.722 INFO:tasks.cephadm:Adding mgr.y on smithi152 2024-04-26T04:50:58.723 INFO:tasks.cephadm:Adding mgr.x on smithi195 2024-04-26T04:50:58.723 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi152=y;smithi195=x' 2024-04-26T04:50:59.010 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[27560]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[27560]: Reconfiguring daemon mon.c on smithi152 2024-04-26T04:50:59.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:59.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:59.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[39324]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:50:59.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:59.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:59.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:59.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:50:59 smithi152 ceph-mon[39324]: Reconfiguring daemon mon.c on smithi152 2024-04-26T04:50:59.471 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:59.471 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:50:59.471 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:59 smithi195 ceph-mon[31427]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:50:59.472 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:59.472 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:59.472 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:59.472 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:50:59 smithi195 ceph-mon[31427]: Reconfiguring daemon mon.c on smithi152 2024-04-26T04:51:00.191 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:00 smithi195 ceph-mon[31427]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:00.191 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:00 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:00.191 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:00 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:00.191 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:00 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:51:00.192 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:00 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:51:00.192 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:00 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[39324]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[27560]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:51:00.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:51:00.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:00 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:01.207 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mgr update... 2024-04-26T04:51:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:01 smithi152 ceph-mon[27560]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:51:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:01 smithi152 ceph-mon[27560]: Reconfiguring daemon mon.b on smithi195 2024-04-26T04:51:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:01 smithi152 ceph-mon[39324]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:51:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:01 smithi152 ceph-mon[39324]: Reconfiguring daemon mon.b on smithi195 2024-04-26T04:51:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:01.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:01 smithi195 ceph-mon[31427]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:51:01.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:01 smithi195 ceph-mon[31427]: Reconfiguring daemon mon.b on smithi195 2024-04-26T04:51:01.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi152=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: Saving service mgr spec with placement smithi152=y;smithi195=x;count:2 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:51:02.271 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:51:02.272 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:51:02.272 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:02 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:02.294 DEBUG:teuthology.orchestra.run.smithi195:mgr.x> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service 2024-04-26T04:51:02.296 INFO:tasks.cephadm:Deploying OSDs... 2024-04-26T04:51:02.296 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:51:02.296 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:51:02.315 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:51:02.315 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_1 2024-04-26T04:51:02.374 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:51:02.374 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:02.375 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-26T04:51:02.375 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:02.375 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:02.375 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-26 04:50:47.954047401 +0000 2024-04-26T04:51:02.375 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-26 04:44:56.762886381 +0000 2024-04-26T04:51:02.375 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-26 04:44:56.762886381 +0000 2024-04-26T04:51:02.375 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-26 04:44:56.762886381 +0000 2024-04-26T04:51:02.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi152=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: Saving service mgr spec with placement smithi152=y;smithi195=x;count:2 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:51:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi152=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: Saving service mgr spec with placement smithi152=y;smithi195=x;count:2 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:51:02.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:51:02.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:51:02.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:02 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:02.408 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-26T04:51:02.408 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-26T04:51:02.408 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000290588 s, 1.8 MB/s 2024-04-26T04:51:02.409 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:51:02.467 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_2 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-26 04:50:47.954047401 +0000 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-26 04:44:57.102882354 +0000 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-26 04:44:57.102882354 +0000 2024-04-26T04:51:02.524 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-26 04:44:57.102882354 +0000 2024-04-26T04:51:02.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:51:02.588 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-26T04:51:02.588 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-26T04:51:02.588 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000216054 s, 2.4 MB/s 2024-04-26T04:51:02.589 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:51:02.649 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_3 2024-04-26T04:51:02.704 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:51:02.704 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:02.704 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-26T04:51:02.704 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:02.704 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:02.704 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-26 04:50:47.955047390 +0000 2024-04-26T04:51:02.704 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-26 04:44:57.436878397 +0000 2024-04-26T04:51:02.705 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-26 04:44:57.436878397 +0000 2024-04-26T04:51:02.705 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-26 04:44:57.436878397 +0000 2024-04-26T04:51:02.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:51:02.767 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-26T04:51:02.767 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-26T04:51:02.767 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000207031 s, 2.5 MB/s 2024-04-26T04:51:02.768 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:51:02.825 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_4 2024-04-26T04:51:02.880 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:51:02.880 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:02.881 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-26T04:51:02.881 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:02.881 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:02.881 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-26 04:50:47.955047390 +0000 2024-04-26T04:51:02.881 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-26 04:44:57.768874464 +0000 2024-04-26T04:51:02.881 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-26 04:44:57.768874464 +0000 2024-04-26T04:51:02.881 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-26 04:44:57.768874464 +0000 2024-04-26T04:51:02.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:51:02.943 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-26T04:51:02.943 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-26T04:51:02.943 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000198868 s, 2.6 MB/s 2024-04-26T04:51:02.944 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:51:03.000 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:51:03.000 DEBUG:teuthology.orchestra.run.smithi195:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:51:03.016 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:51:03.017 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_1 2024-04-26T04:51:03.071 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-04-26 04:50:39.429807072 +0000 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-04-26 04:44:56.757672550 +0000 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-04-26 04:44:56.757672550 +0000 2024-04-26T04:51:03.072 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-04-26 04:44:56.757672550 +0000 2024-04-26T04:51:03.072 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:51:03.135 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-04-26T04:51:03.135 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-04-26T04:51:03.135 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.00010327 s, 5.0 MB/s 2024-04-26T04:51:03.136 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:51:03.192 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_2 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-04-26 04:50:39.429807072 +0000 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-04-26 04:44:57.084666358 +0000 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-04-26 04:44:57.084666358 +0000 2024-04-26T04:51:03.248 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-04-26 04:44:57.084666358 +0000 2024-04-26T04:51:03.249 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:51:03.310 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-04-26T04:51:03.310 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-04-26T04:51:03.311 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000108655 s, 4.7 MB/s 2024-04-26T04:51:03.312 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:51:03.368 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_3 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-04-26 04:50:39.429807072 +0000 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-04-26 04:44:57.402660337 +0000 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-04-26 04:44:57.402660337 +0000 2024-04-26T04:51:03.423 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-04-26 04:44:57.402660337 +0000 2024-04-26T04:51:03.424 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:51:03.483 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:03 smithi195 ceph-mon[31427]: Deploying daemon mgr.x on smithi195 2024-04-26T04:51:03.485 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-04-26T04:51:03.485 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-04-26T04:51:03.485 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000106424 s, 4.8 MB/s 2024-04-26T04:51:03.486 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:51:03.541 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_4 2024-04-26T04:51:03.601 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:51:03.601 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:51:03.601 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-26T04:51:03.601 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:51:03.601 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:51:03.601 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-04-26 04:50:39.430807054 +0000 2024-04-26T04:51:03.601 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-04-26 04:44:57.734654051 +0000 2024-04-26T04:51:03.602 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-04-26 04:44:57.734654051 +0000 2024-04-26T04:51:03.602 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-04-26 04:44:57.734654051 +0000 2024-04-26T04:51:03.602 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:51:03.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:03 smithi152 ceph-mon[27560]: Deploying daemon mgr.x on smithi195 2024-04-26T04:51:03.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:03 smithi152 ceph-mon[39324]: Deploying daemon mgr.x on smithi195 2024-04-26T04:51:03.688 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-04-26T04:51:03.688 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-04-26T04:51:03.688 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 9.2118e-05 s, 5.6 MB/s 2024-04-26T04:51:03.689 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:51:03.704 INFO:tasks.cephadm:Deploying osd.0 on smithi152 with /dev/vg_nvme/lv_4... 2024-04-26T04:51:03.704 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:51:03.950 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:04.132 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 podman[33502]: 2024-04-26 04:51:04.002122305 +0000 UTC m=+0.017497665 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:51:04.327 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:04 smithi152 ceph-mon[27560]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:04.327 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:04 smithi152 ceph-mon[39324]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:04.411 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:04 smithi195 ceph-mon[31427]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:04.412 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 podman[33502]: 2024-04-26 04:51:04.264443552 +0000 UTC m=+0.279818894 container init 0f1dfdd9436ffd0eff02c4360d289369889fdaa7eeb643daa15e6b194f5651fb (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:04.412 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 podman[33502]: 2024-04-26 04:51:04.269927157 +0000 UTC m=+0.285302501 container start 0f1dfdd9436ffd0eff02c4360d289369889fdaa7eeb643daa15e6b194f5651fb (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-26T04:51:04.412 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 ceph-mgr[33539]: -- 172.21.15.195:0/2829059332 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cc685469c0 con 0x55cc6851cc00 2024-04-26T04:51:04.412 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 bash[33502]: 0f1dfdd9436ffd0eff02c4360d289369889fdaa7eeb643daa15e6b194f5651fb 2024-04-26T04:51:04.412 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 systemd[1]: Started Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:51:04.686 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:04.410+0000 7f608dd281c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:51:04.686 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:04.578+0000 7f608dd281c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:51:04.686 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:04.684+0000 7f608dd281c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:51:05.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:05.020+0000 7f608dd281c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:51:05.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:05.102+0000 7f608dd281c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:51:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:05 smithi195 ceph-mon[31427]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:05.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[27560]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:05.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:05.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:05 smithi152 ceph-mon[39324]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:05.909 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:51:05.909 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:51:05.909 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-26T04:51:05.910 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-26T04:51:05.910 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114389 s, 917 MB/s 2024-04-26T04:51:05.910 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-26T04:51:06.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_4 2024-04-26T04:51:06.619 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:06.291+0000 7f608dd281c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:51:06.619 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:06.451+0000 7f608dd281c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:51:06.619 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:06.617+0000 7f608dd281c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:51:06.808 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.808 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.808 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:06.808 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:06.808 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[27560]: Reconfiguring daemon mgr.y on smithi152 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:51:06.809 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:51:06.810 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:51:06.810 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:06.810 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:06 smithi152 ceph-mon[39324]: Reconfiguring daemon mgr.y on smithi152 2024-04-26T04:51:06.876 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.876 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.876 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:06.876 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:06.877 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:06.877 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:51:06.877 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:51:06.877 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:51:06.877 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:06.877 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-mon[31427]: Reconfiguring daemon mgr.y on smithi152 2024-04-26T04:51:07.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:06.875+0000 7f608dd281c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:51:07.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:06.963+0000 7f608dd281c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:51:07.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:07.054+0000 7f608dd281c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:51:07.691 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:07.772 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:07.329+0000 7f608dd281c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:07.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:07.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:07.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:07.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:07.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:07.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:07.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:07.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:08.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:07.787+0000 7f608dd281c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:51:08.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:08.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:08.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:08.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:08.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:08.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:08.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:08.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:08.581 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:08.312+0000 7f608dd281c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:51:08.581 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:08.404+0000 7f608dd281c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:51:08.581 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:08.579+0000 7f608dd281c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:51:09.126 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:08.867+0000 7f608dd281c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:51:09.127 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:09.125+0000 7f608dd281c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:51:09.492 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:09.361+0000 7f608dd281c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:51:09.774 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:09.491+0000 7f608dd281c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:51:09.774 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:09.583+0000 7f608dd281c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:51:09.774 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:09.697+0000 7f608dd281c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:51:10.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:09.785+0000 7f608dd281c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:51:10.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-mon[31427]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:10.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:10.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:10.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:09 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:10.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[27560]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:10.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:10.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:10.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:10.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[39324]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:10.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:10.080 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:10.080 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:09 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:11.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:10 smithi195 ceph-mon[31427]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:11.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:10.605+0000 7f608dd281c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:51:11.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:51:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:51:10.801+0000 7f608dd281c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:51:11.076 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:10 smithi152 ceph-mon[27560]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:11.077 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:10 smithi152 ceph-mon[39324]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[27560]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[27560]: Standby manager daemon x started 2024-04-26T04:51:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:51:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:51:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:51:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:51:12.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[39324]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:12.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[39324]: Standby manager daemon x started 2024-04-26T04:51:12.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:51:12.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:51:12.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:51:12.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:11 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:51:12.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:11 smithi195 ceph-mon[31427]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:12.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:11 smithi195 ceph-mon[31427]: Standby manager daemon x started 2024-04-26T04:51:12.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:11 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:51:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:11 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:51:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:11 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:51:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:11 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1617198925' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:51:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[27560]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-26T04:51:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:51:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/569307183' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2020d0b-1912-45d9-82d2-35664f684411"}]: dispatch 2024-04-26T04:51:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/569307183' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a2020d0b-1912-45d9-82d2-35664f684411"}]': finished 2024-04-26T04:51:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[27560]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:51:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4270398589' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:13.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[39324]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-26T04:51:13.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:51:13.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/569307183' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2020d0b-1912-45d9-82d2-35664f684411"}]: dispatch 2024-04-26T04:51:13.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/569307183' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a2020d0b-1912-45d9-82d2-35664f684411"}]': finished 2024-04-26T04:51:13.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[39324]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:51:13.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:13.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:12 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/4270398589' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:13.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:12 smithi195 ceph-mon[31427]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-26T04:51:13.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:12 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:51:13.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:12 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/569307183' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2020d0b-1912-45d9-82d2-35664f684411"}]: dispatch 2024-04-26T04:51:13.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:12 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/569307183' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a2020d0b-1912-45d9-82d2-35664f684411"}]': finished 2024-04-26T04:51:13.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:12 smithi195 ceph-mon[31427]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:51:13.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:12 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:13.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:12 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/4270398589' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:14.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:14 smithi195 ceph-mon[31427]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:14.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:14 smithi152 ceph-mon[27560]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:14.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:14 smithi152 ceph-mon[39324]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:16.120 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:16 smithi152 ceph-mon[27560]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:16.120 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:16 smithi152 ceph-mon[39324]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:16.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:16 smithi195 ceph-mon[31427]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:18.218 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:18 smithi152 ceph-mon[27560]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:18.219 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:18 smithi152 ceph-mon[39324]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:18 smithi195 ceph-mon[31427]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:19.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:19 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:51:19.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:19 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:19 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:51:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:19 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:19.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:19 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:51:19.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:19 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:20.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:20 smithi195 ceph-mon[31427]: Deploying daemon osd.0 on smithi152 2024-04-26T04:51:20.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:20 smithi195 ceph-mon[31427]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:20.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:20 smithi152 ceph-mon[27560]: Deploying daemon osd.0 on smithi152 2024-04-26T04:51:20.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:20 smithi152 ceph-mon[27560]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:20.375 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:20 smithi152 ceph-mon[39324]: Deploying daemon osd.0 on smithi152 2024-04-26T04:51:20.375 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:20 smithi152 ceph-mon[39324]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:22.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:22 smithi152 ceph-mon[27560]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:22.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:22 smithi152 ceph-mon[39324]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:22.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:22 smithi195 ceph-mon[31427]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:24.116 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[39324]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:24.117 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:24.117 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:24.117 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:24.117 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[27560]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:24.117 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:24.117 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:24.117 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:24 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:24.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:24 smithi195 ceph-mon[31427]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:24.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:24 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:24.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:24 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:24.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:24 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:26.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:26 smithi152 ceph-mon[27560]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:26.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:26 smithi152 ceph-mon[39324]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:26 smithi195 ceph-mon[31427]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:26.921 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 0 on host 'smithi152' 2024-04-26T04:51:27.480 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.480 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.480 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:27.480 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:27.481 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:27.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:27.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:27.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:27.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:27.510 DEBUG:teuthology.orchestra.run.smithi152:osd.0> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.0.service 2024-04-26T04:51:27.513 INFO:tasks.cephadm:Deploying osd.1 on smithi152 with /dev/vg_nvme/lv_3... 2024-04-26T04:51:27.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:51:27.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:27.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:28.251 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:28.484 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[27560]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:51:28.484 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[27560]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:51:28.484 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:28.484 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[27560]: from='osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:28.485 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[27560]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:28.485 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[43636]: 2024-04-26T04:51:28.184+0000 7fcd506c9700 -1 osd.0 0 waiting for initial osdmap 2024-04-26T04:51:28.485 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[43636]: 2024-04-26T04:51:28.186+0000 7fcd4d4c8700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:51:28.485 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[39324]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:51:28.485 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[39324]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:51:28.485 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:28.485 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[39324]: from='osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:28.485 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:28 smithi152 ceph-mon[39324]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:28 smithi195 ceph-mon[31427]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:51:28.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:28 smithi195 ceph-mon[31427]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:51:28.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:28 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:28.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:28 smithi195 ceph-mon[31427]: from='osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:28.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:28 smithi195 ceph-mon[31427]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:29.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:29 smithi195 ceph-mon[31427]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:51:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:29 smithi195 ceph-mon[31427]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:51:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:29 smithi195 ceph-mon[31427]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[27560]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[27560]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[27560]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[39324]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[39324]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:29.604 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:29 smithi152 ceph-mon[39324]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[27560]: osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778] boot 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[27560]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:51:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:30.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[39324]: osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778] boot 2024-04-26T04:51:30.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[39324]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:51:30.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:30 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:30.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:30 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:51:30.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:30 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:51:30.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:30 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:30.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:30 smithi195 ceph-mon[31427]: osd.0 [v2:172.21.15.152:6802/354037778,v1:172.21.15.152:6803/354037778] boot 2024-04-26T04:51:30.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:30 smithi195 ceph-mon[31427]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:51:30.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:30 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:31.314 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi152 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi152 to 16064M 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.315 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi152 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi152 to 16064M 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:31.316 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:31 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi152 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi152 to 16064M 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:31.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:31 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:31.675 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:51:31.675 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:51:31.675 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-26T04:51:31.675 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-26T04:51:31.676 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011525 s, 910 MB/s 2024-04-26T04:51:31.676 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-26T04:51:32.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_3 2024-04-26T04:51:32.572 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:33.928 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:33 smithi152 ceph-mon[27560]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:33.928 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:33 smithi152 ceph-mon[39324]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:34.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:33 smithi195 ceph-mon[31427]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:35.044 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:35.045 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:35.045 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:35.045 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:35.045 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:35.045 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:35.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:35.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:35.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:36.113 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:35 smithi152 ceph-mon[27560]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:36.113 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:35 smithi152 ceph-mon[27560]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:36.114 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:35 smithi152 ceph-mon[39324]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:36.114 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:35 smithi152 ceph-mon[39324]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:36.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:35 smithi195 ceph-mon[31427]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:36.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:35 smithi195 ceph-mon[31427]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:38.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[27560]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:38.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/305802980' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]: dispatch 2024-04-26T04:51:38.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]: dispatch 2024-04-26T04:51:38.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]': finished 2024-04-26T04:51:38.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[27560]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:51:38.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3444273222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[39324]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/305802980' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]: dispatch 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]: dispatch 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]': finished 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[39324]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:38.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:37 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3444273222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:37 smithi195 ceph-mon[31427]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:37 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/305802980' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]: dispatch 2024-04-26T04:51:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:37 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]: dispatch 2024-04-26T04:51:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:37 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e15a8724-faa2-4271-bb15-5cc3d6886ee7"}]': finished 2024-04-26T04:51:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:37 smithi195 ceph-mon[31427]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:51:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:37 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:37 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3444273222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:40.074 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:39 smithi152 ceph-mon[27560]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:40.075 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:39 smithi152 ceph-mon[39324]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:40.082 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:39 smithi195 ceph-mon[31427]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:42.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:41 smithi152 ceph-mon[27560]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:42.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:41 smithi152 ceph-mon[39324]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:42.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:41 smithi195 ceph-mon[31427]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:43.867 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:43 smithi152 ceph-mon[39324]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:43.868 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:51:43.868 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:43.868 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:43 smithi152 ceph-mon[27560]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:43.868 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:51:43.868 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:44.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:43 smithi195 ceph-mon[31427]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:44.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:51:44.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:45.129 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:44 smithi152 ceph-mon[39324]: Deploying daemon osd.1 on smithi152 2024-04-26T04:51:45.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:44 smithi152 ceph-mon[27560]: Deploying daemon osd.1 on smithi152 2024-04-26T04:51:45.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:44 smithi195 ceph-mon[31427]: Deploying daemon osd.1 on smithi152 2024-04-26T04:51:45.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:45 smithi152 ceph-mon[27560]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:45.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:45 smithi152 ceph-mon[39324]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:46.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:45 smithi195 ceph-mon[31427]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:48.087 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:47 smithi152 ceph-mon[27560]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:48.087 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:47 smithi152 ceph-mon[39324]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:48.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:47 smithi195 ceph-mon[31427]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:49.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:49.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:49.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:49.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[27560]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:49.362 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:49.362 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:49.363 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:49.363 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:49 smithi152 ceph-mon[39324]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:49.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:49 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:49.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:49 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:49.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:49 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:49.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:49 smithi195 ceph-mon[31427]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:51.957 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 1 on host 'smithi152' 2024-04-26T04:51:52.262 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:52.262 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.262 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.262 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.262 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.263 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.264 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.264 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:51:52.264 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:51:52.264 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.264 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.264 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:51:52.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:51:52.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:52.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.549 DEBUG:teuthology.orchestra.run.smithi152:osd.1> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.1.service 2024-04-26T04:51:52.551 INFO:tasks.cephadm:Deploying osd.2 on smithi152 with /dev/vg_nvme/lv_2... 2024-04-26T04:51:52.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:51:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[27560]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:51:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[27560]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:51:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[27560]: from='osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[27560]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:53.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[39324]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:51:53.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[39324]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:51:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[39324]: from='osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-mon[39324]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:53.131 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[50026]: 2024-04-26T04:51:52.972+0000 7fa4c5571700 -1 osd.1 0 waiting for initial osdmap 2024-04-26T04:51:53.131 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:51:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[50026]: 2024-04-26T04:51:52.975+0000 7fa4c2370700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:51:53.252 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:52 smithi195 ceph-mon[31427]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:52 smithi195 ceph-mon[31427]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:52 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:52 smithi195 ceph-mon[31427]: from='osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:53.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:52 smithi195 ceph-mon[31427]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:51:54.149 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[27560]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:54.149 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[27560]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:51:54.150 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[27560]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:51:54.150 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:54.150 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:54.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[39324]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:54.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[39324]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:51:54.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[39324]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:51:54.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:54.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:54.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:53 smithi195 ceph-mon[31427]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:54.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:53 smithi195 ceph-mon[31427]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:51:54.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:53 smithi195 ceph-mon[31427]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:51:54.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:54.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:55.206 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:51:55.206 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:51:55.206 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[39324]: osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910] boot 2024-04-26T04:51:55.206 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[39324]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:51:55.206 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:55.207 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:51:55.207 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:51:55.207 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[27560]: osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910] boot 2024-04-26T04:51:55.207 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[27560]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:51:55.207 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:54 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:55.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:54 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:51:55.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:54 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:51:55.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:54 smithi195 ceph-mon[31427]: osd.1 [v2:172.21.15.152:6810/1928910910,v1:172.21.15.152:6811/1928910910] boot 2024-04-26T04:51:55.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:54 smithi195 ceph-mon[31427]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:51:55.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:54 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:56.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:56.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:51:56.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:56.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:51:56.700 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:51:56.701 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:51:56.701 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-26T04:51:56.701 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-26T04:51:56.701 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133971 s, 783 MB/s 2024-04-26T04:51:56.701 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-26T04:51:57.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:56 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi152 2024-04-26T04:51:57.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:56 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi152 to 8032M 2024-04-26T04:51:57.301 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:56 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi152 2024-04-26T04:51:57.301 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:56 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi152 to 8032M 2024-04-26T04:51:57.301 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:56 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi152 2024-04-26T04:51:57.302 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:56 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi152 to 8032M 2024-04-26T04:51:57.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_2 2024-04-26T04:51:57.587 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:58.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:57 smithi195 ceph-mon[31427]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:58.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:57 smithi152 ceph-mon[27560]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:57 smithi152 ceph-mon[39324]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[27560]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[39324]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:00.072 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:51:59 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:59 smithi195 ceph-mon[31427]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:51:59 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:01.093 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:00 smithi152 ceph-mon[27560]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:01.094 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:00 smithi152 ceph-mon[39324]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:01.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:00 smithi195 ceph-mon[31427]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:02.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:01 smithi195 ceph-mon[31427]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:02.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:01 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1057886380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]: dispatch 2024-04-26T04:52:02.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:01 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]: dispatch 2024-04-26T04:52:02.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:01 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]': finished 2024-04-26T04:52:02.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:01 smithi195 ceph-mon[31427]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:52:02.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[27560]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1057886380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]: dispatch 2024-04-26T04:52:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]: dispatch 2024-04-26T04:52:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]': finished 2024-04-26T04:52:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[27560]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:52:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[39324]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1057886380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]: dispatch 2024-04-26T04:52:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]: dispatch 2024-04-26T04:52:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3d3029a0-ea36-41c5-b415-20c0ff079f0e"}]': finished 2024-04-26T04:52:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[39324]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:52:02.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:03.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:03 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1203665301' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:03.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:03 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1203665301' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:03 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1203665301' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:04.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:04 smithi195 ceph-mon[31427]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:04.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:04 smithi152 ceph-mon[27560]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:04.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:04 smithi152 ceph-mon[39324]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:06.209 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:06 smithi152 ceph-mon[27560]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:06.210 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:06 smithi152 ceph-mon[39324]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:06.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:06 smithi195 ceph-mon[31427]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:08.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:08 smithi195 ceph-mon[31427]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:08.288 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:08 smithi152 ceph-mon[27560]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:08.288 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:08 smithi152 ceph-mon[39324]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:09.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:09 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:52:09.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:09 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:09 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:52:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:09 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:09.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:09 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:52:09.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:09 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:10.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:10 smithi195 ceph-mon[31427]: Deploying daemon osd.2 on smithi152 2024-04-26T04:52:10.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:10 smithi195 ceph-mon[31427]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:10.360 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:10 smithi152 ceph-mon[27560]: Deploying daemon osd.2 on smithi152 2024-04-26T04:52:10.360 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:10 smithi152 ceph-mon[27560]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:10.360 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:10 smithi152 ceph-mon[39324]: Deploying daemon osd.2 on smithi152 2024-04-26T04:52:10.360 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:10 smithi152 ceph-mon[39324]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:12.106 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:12 smithi152 ceph-mon[27560]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:12.106 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:12 smithi152 ceph-mon[39324]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:12.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:12 smithi195 ceph-mon[31427]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:14.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:14 smithi195 ceph-mon[31427]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:14.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:14 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:14.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:14 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:14.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:14 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:14.310 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[39324]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:14.311 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:14.311 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:14.311 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:14.311 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[27560]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:14.311 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:14.311 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:14.311 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:14 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:16.118 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:16 smithi152 ceph-mon[39324]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:16.118 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:16 smithi152 ceph-mon[27560]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:16.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:16 smithi195 ceph-mon[31427]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:17.118 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 2 on host 'smithi152' 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:52:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:17 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:17.767 DEBUG:teuthology.orchestra.run.smithi152:osd.2> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.2.service 2024-04-26T04:52:17.770 INFO:tasks.cephadm:Deploying osd.3 on smithi152 with /dev/vg_nvme/lv_1... 2024-04-26T04:52:17.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:52:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:17.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:52:17.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:52:17.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:17.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:17 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:18.481 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:18.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[27560]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:52:18.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[27560]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:52:18.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:18.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[27560]: from='osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:18.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[27560]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:18.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[39324]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:52:18.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[39324]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:52:18.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:18.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[39324]: from='osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:18.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-mon[39324]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:18.624 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[56517]: 2024-04-26T04:52:18.363+0000 7fe1ea3b8700 -1 osd.2 0 waiting for initial osdmap 2024-04-26T04:52:18.624 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[56517]: 2024-04-26T04:52:18.366+0000 7fe1e31af700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:52:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:18 smithi195 ceph-mon[31427]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:52:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:18 smithi195 ceph-mon[31427]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:52:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:18 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:18 smithi195 ceph-mon[31427]: from='osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:18 smithi195 ceph-mon[31427]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[27560]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[27560]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[27560]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[39324]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[39324]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:52:19.594 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:19.595 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:19.595 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:19 smithi152 ceph-mon[39324]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:19 smithi195 ceph-mon[31427]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:52:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:19 smithi195 ceph-mon[31427]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:52:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:19 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:19 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:19 smithi195 ceph-mon[31427]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:52:20.602 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:52:20.602 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:52:20.602 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[39324]: osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721] boot 2024-04-26T04:52:20.602 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[39324]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:52:20.602 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:20.603 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:52:20.603 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:52:20.603 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[27560]: osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721] boot 2024-04-26T04:52:20.603 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[27560]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:52:20.603 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:20 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:20.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:20 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:52:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:20 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:52:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:20 smithi195 ceph-mon[31427]: osd.2 [v2:172.21.15.152:6818/2274658721,v1:172.21.15.152:6819/2274658721] boot 2024-04-26T04:52:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:20 smithi195 ceph-mon[31427]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:52:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:20 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:52:21.570 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:52:21.570 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:52:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-26T04:52:21.570 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-26T04:52:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129882 s, 807 MB/s 2024-04-26T04:52:21.570 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-26T04:52:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:21 smithi195 ceph-mon[31427]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:52:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:21 smithi195 ceph-mon[31427]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:21 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:52:21.876 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:21 smithi152 ceph-mon[27560]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:52:21.877 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:21 smithi152 ceph-mon[27560]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:21.877 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:52:21.877 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:21 smithi152 ceph-mon[39324]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:52:21.877 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:21 smithi152 ceph-mon[39324]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:21.877 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:21 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:52:22.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:52:22.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:52:22.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:52:22.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi152 2024-04-26T04:52:22.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi152 to 5354M 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:52:22.507 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi152 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi152 to 5354M 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:22.508 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_1 2024-04-26T04:52:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi152 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi152 to 5354M 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:22.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:22.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:22.810 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:23.209 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60882]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:52:23.209 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60882]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:23.209 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60882]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:23.210 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60882]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[27560]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[27560]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[39324]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[39324]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:23.562 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:23 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:23.563 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60923]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:52:23.563 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60923]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:23.563 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60923]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:23.563 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60923]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:23.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:23 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:52:23.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:23 smithi195 ceph-mon[31427]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:52:23.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:23 smithi195 ceph-mon[31427]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:23.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:23 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:23.821 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60926]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:52:23.821 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60926]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:23.821 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60926]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:23.821 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60926]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:24.083 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60936]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:52:24.083 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60936]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:24.084 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60936]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:24.084 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 04:52:23 smithi152 sudo[60936]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:24.084 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60960]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:52:24.084 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60960]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:24.084 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60960]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:24.084 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60960]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:24.333 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60982]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:52:24.334 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60982]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:24.334 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60982]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:24.334 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[60982]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:24.842 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61012]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:52:24.842 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61012]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:24.843 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61012]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:24.843 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61012]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:25.104 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:24 smithi195 ceph-mon[31427]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:52:25.105 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:24 smithi195 ceph-mon[31427]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:52:25.105 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:24 smithi195 ceph-mon[31427]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:25.105 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34267]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:52:25.105 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34267]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:25.105 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34267]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:25.105 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34267]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:24 smithi152 ceph-mon[39324]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:24 smithi152 ceph-mon[39324]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:24 smithi152 ceph-mon[39324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61016]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61016]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61016]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 sudo[61016]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:25.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 ceph-mon[27560]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:52:25.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 ceph-mon[27560]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:52:25.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:24 smithi152 ceph-mon[27560]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:25.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34270]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:52:25.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34270]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:25.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34270]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:25.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 sudo[34270]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:25.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61178]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:52:25.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61178]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:25.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61178]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:25.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61178]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:26.274 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:26.274 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:26.274 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:26.275 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61181]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61181]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61181]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 sudo[61181]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.276 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:27.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:26 smithi195 ceph-mon[31427]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:27.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:26 smithi195 ceph-mon[31427]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:27.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:26 smithi152 ceph-mon[27560]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:27.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:26 smithi152 ceph-mon[27560]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:27.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:26 smithi152 ceph-mon[39324]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:52:27.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:26 smithi152 ceph-mon[39324]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:52:28.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:27 smithi195 ceph-mon[31427]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:28.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:27 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2356886122' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "482b6933-60c5-4a4f-a4e9-8365825c20b7"}]: dispatch 2024-04-26T04:52:28.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:27 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2356886122' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "482b6933-60c5-4a4f-a4e9-8365825c20b7"}]': finished 2024-04-26T04:52:28.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:27 smithi195 ceph-mon[31427]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:52:28.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[39324]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2356886122' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "482b6933-60c5-4a4f-a4e9-8365825c20b7"}]: dispatch 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2356886122' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "482b6933-60c5-4a4f-a4e9-8365825c20b7"}]': finished 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[39324]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[27560]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2356886122' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "482b6933-60c5-4a4f-a4e9-8365825c20b7"}]: dispatch 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2356886122' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "482b6933-60c5-4a4f-a4e9-8365825c20b7"}]': finished 2024-04-26T04:52:28.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[27560]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:52:28.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:29.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:28 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3442115878' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:29.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:28 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3442115878' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:28 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3442115878' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:30.248 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:29 smithi152 ceph-mon[27560]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:30.249 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:29 smithi152 ceph-mon[39324]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:30.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:29 smithi195 ceph-mon[31427]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:32.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:31 smithi152 ceph-mon[27560]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T04:52:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:31 smithi152 ceph-mon[39324]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T04:52:32.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:31 smithi195 ceph-mon[31427]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T04:52:34.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:33 smithi195 ceph-mon[31427]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:52:34.323 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:33 smithi152 ceph-mon[27560]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:52:34.324 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:33 smithi152 ceph-mon[39324]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:52:35.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:52:35.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:35.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:34 smithi195 ceph-mon[31427]: Deploying daemon osd.3 on smithi152 2024-04-26T04:52:35.305 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:52:35.305 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:35.305 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:34 smithi152 ceph-mon[27560]: Deploying daemon osd.3 on smithi152 2024-04-26T04:52:35.305 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:52:35.306 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:35.306 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:34 smithi152 ceph-mon[39324]: Deploying daemon osd.3 on smithi152 2024-04-26T04:52:36.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:35 smithi195 ceph-mon[31427]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:52:36.320 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:35 smithi152 ceph-mon[27560]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:52:36.320 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:35 smithi152 ceph-mon[39324]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:52:38.195 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:37 smithi152 ceph-mon[27560]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:52:38.195 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:37 smithi152 ceph-mon[39324]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:52:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:37 smithi195 ceph-mon[31427]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:52:40.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[39324]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2024-04-26T04:52:40.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:40.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:40.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[27560]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2024-04-26T04:52:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:40.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:40 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:40 smithi195 ceph-mon[31427]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2024-04-26T04:52:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:40 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:42 smithi152 ceph-mon[27560]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:52:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:42 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:42 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:42 smithi152 ceph-mon[39324]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:52:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:42 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:42 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:42.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:42 smithi195 ceph-mon[31427]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:52:42.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:42 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:42.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:42 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:42.806 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 3 on host 'smithi152' 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:43.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:52:43.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:52:43.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:43.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:43.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:43.497 DEBUG:teuthology.orchestra.run.smithi152:osd.3> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.3.service 2024-04-26T04:52:43.500 INFO:tasks.cephadm:Deploying osd.4 on smithi195 with /dev/vg_nvme/lv_4... 2024-04-26T04:52:43.500 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:43.527 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:43.754 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[27560]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[27560]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:52:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[27560]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:52:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[27560]: from='osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[27560]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:44.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[39324]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:44.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[39324]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:52:44.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[39324]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:52:44.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:44.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[39324]: from='osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:44.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-mon[39324]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:44.381 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[63099]: 2024-04-26T04:52:44.134+0000 7f79c678b700 -1 osd.3 0 waiting for initial osdmap 2024-04-26T04:52:44.381 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 04:52:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[63099]: 2024-04-26T04:52:44.138+0000 7f79c1586700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:52:44.391 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:44 smithi195 ceph-mon[31427]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:44.391 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:44 smithi195 ceph-mon[31427]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:52:44.391 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:44 smithi195 ceph-mon[31427]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:52:44.391 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:44 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:44.391 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:44 smithi195 ceph-mon[31427]: from='osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:44.391 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:44 smithi195 ceph-mon[31427]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T04:52:45.467 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:52:45.467 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:52:45.467 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-04-26T04:52:45.467 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-04-26T04:52:45.467 INFO:teuthology.orchestra.run.smithi195.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00870075 s, 1.2 GB/s 2024-04-26T04:52:45.467 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 2024-04-26T04:52:45.467 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-04-26T04:52:45.468 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:45 smithi195 ceph-mon[31427]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:52:45.468 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:45 smithi195 ceph-mon[31427]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:52:45.468 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.468 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.468 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[27560]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:52:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[27560]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:52:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[39324]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-26T04:52:45.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[39324]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:52:45.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:45 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:45.962 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_4 2024-04-26T04:52:46.248 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:46.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:46 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:52:46.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:46 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:52:46.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:46 smithi195 ceph-mon[31427]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:46.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:46 smithi195 ceph-mon[31427]: osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688] boot 2024-04-26T04:52:46.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:46 smithi195 ceph-mon[31427]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:52:46.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:46 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[27560]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[27560]: osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688] boot 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[27560]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:52:46.350 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:52:46.351 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[39324]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:52:46.351 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[39324]: osd.3 [v2:172.21.15.152:6826/816595688,v1:172.21.15.152:6827/816595688] boot 2024-04-26T04:52:46.351 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[39324]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:52:46.351 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:46 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:52:47.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:47.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:47 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:52:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:47.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:47.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:52:47.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:47.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:47.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:47.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:47.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:47 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:52:48.415 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:48 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi152 2024-04-26T04:52:48.416 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:48 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi152 to 4016M 2024-04-26T04:52:48.416 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:48 smithi195 ceph-mon[31427]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:48.416 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:48 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:48.416 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:48 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:48.416 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:48 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi152 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi152 to 4016M 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[27560]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi152 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi152 to 4016M 2024-04-26T04:52:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[39324]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:48.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:48.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:48.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:48 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:49.395 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:49 smithi195 ceph-mon[31427]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:49.542 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:49 smithi152 ceph-mon[27560]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:49.542 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:49 smithi152 ceph-mon[39324]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:50 smithi195 ceph-mon[31427]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:50 smithi152 ceph-mon[27560]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:50.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:50 smithi152 ceph-mon[39324]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:51.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:51 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/2472876142' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]: dispatch 2024-04-26T04:52:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:51 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]: dispatch 2024-04-26T04:52:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:51 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]': finished 2024-04-26T04:52:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:51 smithi195 ceph-mon[31427]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:52:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:51 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:51 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/2267638145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/2472876142' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]: dispatch 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]: dispatch 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]': finished 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[27560]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/2267638145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/2472876142' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]: dispatch 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]: dispatch 2024-04-26T04:52:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f134850-9241-4b01-9333-d5a46084c312"}]': finished 2024-04-26T04:52:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[39324]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:52:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:51 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/2267638145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:52.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:52 smithi195 ceph-mon[31427]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:52.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:52 smithi152 ceph-mon[27560]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:52 smithi152 ceph-mon[39324]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:54.481 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:54 smithi195 ceph-mon[31427]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:54 smithi152 ceph-mon[27560]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:54 smithi152 ceph-mon[39324]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:56.449 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:56 smithi195 ceph-mon[31427]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:56 smithi152 ceph-mon[27560]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:56.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:56 smithi152 ceph-mon[39324]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:57.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:57 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:52:57.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:57 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:57.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:57 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:52:57.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:57 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:57 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:52:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:57 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:58.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:58 smithi195 ceph-mon[31427]: Deploying daemon osd.4 on smithi195 2024-04-26T04:52:58.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:58 smithi195 ceph-mon[31427]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:58.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:58 smithi152 ceph-mon[27560]: Deploying daemon osd.4 on smithi195 2024-04-26T04:52:58.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:58 smithi152 ceph-mon[27560]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:58 smithi152 ceph-mon[39324]: Deploying daemon osd.4 on smithi195 2024-04-26T04:52:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:58 smithi152 ceph-mon[39324]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:59.489 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:52:59 smithi195 ceph-mon[31427]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:59.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:52:59 smithi152 ceph-mon[27560]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:59.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:52:59 smithi152 ceph-mon[39324]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:01.966 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:01 smithi195 ceph-mon[31427]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:01.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:01.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:01.967 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:02.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[27560]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:02.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:02.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:02.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:02.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[39324]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:02.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:02.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:02.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:04.002 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:03 smithi195 ceph-mon[31427]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:04.002 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:03 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:04.002 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:03 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:04.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:03 smithi152 ceph-mon[27560]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:04.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:03 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:04.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:03 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:04.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:03 smithi152 ceph-mon[39324]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:04.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:03 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:04.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:03 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:04.670 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 4 on host 'smithi195' 2024-04-26T04:53:05.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:05.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:05.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:53:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:53:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:04 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:05.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:05.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:05.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:53:05.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:53:05.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:05.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:04 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:05.316 DEBUG:teuthology.orchestra.run.smithi195:osd.4> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.4.service 2024-04-26T04:53:05.318 INFO:tasks.cephadm:Deploying osd.5 on smithi195 with /dev/vg_nvme/lv_3... 2024-04-26T04:53:05.318 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:53:06.044 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:06.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-mon[31427]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-mon[31427]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:53:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-mon[31427]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:53:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-mon[31427]: from='osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-mon[31427]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:06.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T04:53:06.162+0000 7f71f2adb700 -1 osd.4 0 waiting for initial osdmap 2024-04-26T04:53:06.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 04:53:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T04:53:06.168+0000 7f71ed8d6700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:53:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[27560]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[27560]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:53:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[27560]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:53:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[27560]: from='osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[27560]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:06.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[39324]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:06.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[39324]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:53:06.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[39324]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:53:06.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:06.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[39324]: from='osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:06.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:06 smithi152 ceph-mon[39324]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:07.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:07 smithi195 ceph-mon[31427]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:07.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:07 smithi195 ceph-mon[31427]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:53:07.424 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:07.425 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:07 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[27560]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[27560]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[39324]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[39324]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:07.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:07 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:08.261 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:08 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:53:08.262 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:08 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:53:08.262 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:08 smithi195 ceph-mon[31427]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:08.262 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:08 smithi195 ceph-mon[31427]: osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031] boot 2024-04-26T04:53:08.262 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:08 smithi195 ceph-mon[31427]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:53:08.262 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:08 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:08.262 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:08 smithi195 ceph-mon[31427]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:53:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:53:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[27560]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[27560]: osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031] boot 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[27560]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[27560]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[39324]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[39324]: osd.4 [v2:172.21.15.195:6800/3775845031,v1:172.21.15.195:6801/3775845031] boot 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[39324]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:08 smithi152 ceph-mon[39324]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:53:08.754 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:53:08.754 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:53:08.754 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-04-26T04:53:08.755 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-04-26T04:53:08.755 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0100702 s, 1.0 GB/s 2024-04-26T04:53:08.755 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-04-26T04:53:09.703 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_3 2024-04-26T04:53:09.948 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:10.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:10.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi195 2024-04-26T04:53:10.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:10.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi195 to 17088M 2024-04-26T04:53:10.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:10.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:53:10.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:10.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:10.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi195 2024-04-26T04:53:10.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:10.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi195 to 17088M 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi195 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:10.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi195 to 17088M 2024-04-26T04:53:10.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:10.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:10.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:53:10.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:10.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:11.278 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:11 smithi195 ceph-mon[31427]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:53:11.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:11 smithi152 ceph-mon[27560]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:53:11.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:11 smithi152 ceph-mon[39324]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:53:12.475 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:12 smithi195 ceph-mon[31427]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:53:12.475 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:12 smithi195 ceph-mon[31427]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:53:12.475 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:12 smithi195 ceph-mon[31427]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:12.475 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:12 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:12.475 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:12 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:12.475 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:12 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[27560]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:53:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[27560]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:53:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[27560]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:12.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[39324]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:53:12.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[39324]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:53:12.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[39324]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:12.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:12.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:12.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:12 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:13.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:13 smithi195 ceph-mon[31427]: pgmap v118: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-04-26T04:53:13.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:13 smithi152 ceph-mon[27560]: pgmap v118: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-04-26T04:53:13.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:13 smithi152 ceph-mon[39324]: pgmap v118: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-04-26T04:53:14.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:14 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/3998337202' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]: dispatch 2024-04-26T04:53:14.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:14 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]: dispatch 2024-04-26T04:53:14.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:14 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]': finished 2024-04-26T04:53:14.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:14 smithi195 ceph-mon[31427]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:53:14.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:14 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:14.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/3998337202' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]: dispatch 2024-04-26T04:53:14.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]: dispatch 2024-04-26T04:53:14.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]': finished 2024-04-26T04:53:14.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[27560]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:53:14.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:14.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/3998337202' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]: dispatch 2024-04-26T04:53:14.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]: dispatch 2024-04-26T04:53:14.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab6ff913-82f5-4abf-a461-cb0a7216ef01"}]': finished 2024-04-26T04:53:14.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[39324]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:53:14.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:14 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:15.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:15 smithi195 ceph-mon[31427]: pgmap v120: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-26T04:53:15.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:15 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/4043450482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:15.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:15 smithi152 ceph-mon[27560]: pgmap v120: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-26T04:53:15.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:15 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/4043450482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:15.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:15 smithi152 ceph-mon[39324]: pgmap v120: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-26T04:53:15.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:15 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/4043450482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:18.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:17 smithi195 ceph-mon[31427]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:18.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:17 smithi152 ceph-mon[27560]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:18.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:17 smithi152 ceph-mon[39324]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:20.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:19 smithi195 ceph-mon[31427]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:20.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:19 smithi152 ceph-mon[27560]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:20.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:19 smithi152 ceph-mon[39324]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:21.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:20 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:53:21.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:20 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:20 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:53:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:20 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:21.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:20 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:53:21.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:20 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:21.997 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:21 smithi195 ceph-mon[31427]: Deploying daemon osd.5 on smithi195 2024-04-26T04:53:21.997 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:21 smithi195 ceph-mon[31427]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:22.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:21 smithi152 ceph-mon[27560]: Deploying daemon osd.5 on smithi195 2024-04-26T04:53:22.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:21 smithi152 ceph-mon[27560]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:22.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:21 smithi152 ceph-mon[39324]: Deploying daemon osd.5 on smithi195 2024-04-26T04:53:22.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:21 smithi152 ceph-mon[39324]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:23.915 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:23 smithi195 ceph-mon[31427]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:24.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:23 smithi152 ceph-mon[27560]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:24.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:23 smithi152 ceph-mon[39324]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:26.100 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:25 smithi195 ceph-mon[31427]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:26.101 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:26.101 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:26.101 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:25 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:26.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[27560]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:26.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:26.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:26.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:26.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[39324]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:26.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:26.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:26.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:25 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:27.902 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:27 smithi195 ceph-mon[31427]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:27.902 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:27 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:28.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:27 smithi152 ceph-mon[27560]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:28.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:27 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:28.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:27 smithi152 ceph-mon[39324]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:28.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:27 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:28.913 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:28 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:28.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:28 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:28.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:28 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:28.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:28 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:28.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:28 smithi195 ceph-mon[31427]: from='osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:53:28.914 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:28 smithi195 ceph-mon[31427]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:53:28.962 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 5 on host 'smithi195' 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[27560]: from='osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[27560]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:29.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:29.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:29.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[39324]: from='osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:53:29.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:28 smithi152 ceph-mon[39324]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:53:29.566 DEBUG:teuthology.orchestra.run.smithi195:osd.5> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.5.service 2024-04-26T04:53:29.568 INFO:tasks.cephadm:Deploying osd.6 on smithi195 with /dev/vg_nvme/lv_2... 2024-04-26T04:53:29.569 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:53:30.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:30.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:53:30.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:53:30.078 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:30.079 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: from='osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:30.079 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:30.079 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:30.079 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:30.079 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:30.079 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T04:53:29.863+0000 7f41f146d700 -1 osd.5 0 waiting for initial osdmap 2024-04-26T04:53:30.079 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 04:53:29 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T04:53:29.868+0000 7f41ea264700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: from='osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: from='osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:30.262 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:30 smithi195 ceph-mon[31427]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:30 smithi195 ceph-mon[31427]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:53:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:30 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:30 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:30 smithi195 ceph-mon[31427]: osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311] boot 2024-04-26T04:53:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:30 smithi195 ceph-mon[31427]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:53:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:30 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[27560]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[27560]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[27560]: osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311] boot 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[27560]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[39324]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:31.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[39324]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:53:31.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:31.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[39324]: osd.5 [v2:172.21.15.195:6808/3668288311,v1:172.21.15.195:6809/3668288311] boot 2024-04-26T04:53:31.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[39324]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:53:31.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:30 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:32.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:53:32.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:53:32.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[27560]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:32.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[27560]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:53:32.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[27560]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:53:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:53:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:53:32.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[39324]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:32.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[39324]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:53:32.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:31 smithi152 ceph-mon[39324]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:53:32.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:31 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:53:32.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:31 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:53:32.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:31 smithi195 ceph-mon[31427]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:53:32.141 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:31 smithi195 ceph-mon[31427]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:53:32.142 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:31 smithi195 ceph-mon[31427]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:53:32.985 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:53:32.986 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:53:32.986 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-04-26T04:53:32.986 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-04-26T04:53:32.986 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00874909 s, 1.2 GB/s 2024-04-26T04:53:32.986 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-04-26T04:53:33.240 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:33 smithi195 ceph-mon[31427]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:53:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:33 smithi152 ceph-mon[27560]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:53:33.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:33 smithi152 ceph-mon[39324]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:53:33.903 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_2 2024-04-26T04:53:34.151 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:34.270 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:34.270 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:53:34.270 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.274 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.274 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:34.274 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:34.274 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.274 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:34.274 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:34.274 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:34 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:34 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:35.108 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:35 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi195 2024-04-26T04:53:35.108 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:35 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi195 to 8544M 2024-04-26T04:53:35.108 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:35 smithi195 ceph-mon[31427]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:53:35.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:35 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi195 2024-04-26T04:53:35.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:35 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi195 to 8544M 2024-04-26T04:53:35.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:35 smithi152 ceph-mon[27560]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:53:35.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:35 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi195 2024-04-26T04:53:35.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:35 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi195 to 8544M 2024-04-26T04:53:35.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:35 smithi152 ceph-mon[39324]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:53:36.333 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:36 smithi195 ceph-mon[31427]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:36.333 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:36 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:36.333 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:36 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:36.333 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:36 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:36.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[27560]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[39324]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:36 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:37.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:37 smithi195 ceph-mon[31427]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:37.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:37 smithi152 ceph-mon[27560]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:37.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:37 smithi152 ceph-mon[39324]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:38.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:38 smithi152 ceph-mon[27560]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-26T04:53:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:38 smithi152 ceph-mon[27560]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:53:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:38 smithi152 ceph-mon[27560]: Cluster is now healthy 2024-04-26T04:53:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:38 smithi152 ceph-mon[39324]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-26T04:53:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:38 smithi152 ceph-mon[39324]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:53:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:38 smithi152 ceph-mon[39324]: Cluster is now healthy 2024-04-26T04:53:38.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:38 smithi195 ceph-mon[31427]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-26T04:53:38.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:38 smithi195 ceph-mon[31427]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:53:38.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:38 smithi195 ceph-mon[31427]: Cluster is now healthy 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/2873131575' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]: dispatch 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]: dispatch 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]': finished 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[27560]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/1558385378' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/2873131575' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]: dispatch 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]: dispatch 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]': finished 2024-04-26T04:53:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[39324]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:53:39.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:39.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:39 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/1558385378' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:39.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:39 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/2873131575' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]: dispatch 2024-04-26T04:53:39.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:39 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]: dispatch 2024-04-26T04:53:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:39 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c937359c-e6b6-4e0e-afc7-236848f8cd81"}]': finished 2024-04-26T04:53:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:39 smithi195 ceph-mon[31427]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:53:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:39 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:39 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/1558385378' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:40.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:40 smithi152 ceph-mon[39324]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:40 smithi152 ceph-mon[27560]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:40.431 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:40 smithi195 ceph-mon[31427]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:42.362 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:42 smithi195 ceph-mon[31427]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-26T04:53:42.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:42 smithi152 ceph-mon[27560]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-26T04:53:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:42 smithi152 ceph-mon[39324]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-26T04:53:44.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:44 smithi195 ceph-mon[31427]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-26T04:53:44.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:44 smithi152 ceph-mon[27560]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-26T04:53:44.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:44 smithi152 ceph-mon[39324]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-26T04:53:45.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:45 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:53:45.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:45 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:45.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:53:45.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:45.446 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:53:45.446 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:46.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:46 smithi152 ceph-mon[39324]: Deploying daemon osd.6 on smithi195 2024-04-26T04:53:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:46 smithi152 ceph-mon[39324]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:46 smithi152 ceph-mon[27560]: Deploying daemon osd.6 on smithi195 2024-04-26T04:53:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:46 smithi152 ceph-mon[27560]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:46.410 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:46 smithi195 ceph-mon[31427]: Deploying daemon osd.6 on smithi195 2024-04-26T04:53:46.411 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:46 smithi195 ceph-mon[31427]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:48.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:48 smithi152 ceph-mon[27560]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:48.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:48 smithi152 ceph-mon[39324]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:48 smithi195 ceph-mon[31427]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[27560]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[39324]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:50.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:50 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:50 smithi195 ceph-mon[31427]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:50 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:50 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:50 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:52 smithi195 ceph-mon[31427]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:52.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:52 smithi152 ceph-mon[27560]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:52.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:52 smithi152 ceph-mon[39324]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:53.078 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 6 on host 'smithi195' 2024-04-26T04:53:53.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.451 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:53.451 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:53.451 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.451 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:53:53.451 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:53:53.451 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:53.452 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.452 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.452 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:53 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:53.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:53:53.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:53.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:53.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:53 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.734 DEBUG:teuthology.orchestra.run.smithi195:osd.6> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.6.service 2024-04-26T04:53:53.736 INFO:tasks.cephadm:Deploying osd.7 on smithi195 with /dev/vg_nvme/lv_1... 2024-04-26T04:53:53.736 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:53:54.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:54 smithi195 ceph-mon[31427]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:53:54.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:54 smithi195 ceph-mon[31427]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:53:54.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:54 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:54.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:54 smithi195 ceph-mon[31427]: from='osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:54.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:54 smithi195 ceph-mon[31427]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:54.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 04:53:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T04:53:54.250+0000 7fe465dd3700 -1 osd.6 0 waiting for initial osdmap 2024-04-26T04:53:54.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 04:53:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T04:53:54.257+0000 7fe45f3cb700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:53:54.580 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[27560]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[27560]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[27560]: from='osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[27560]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[39324]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[39324]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:53:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:54.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[39324]: from='osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:54.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:54 smithi152 ceph-mon[39324]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:53:55.519 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:55 smithi195 ceph-mon[31427]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:55.519 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:55 smithi195 ceph-mon[31427]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:53:55.520 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:55.520 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:55 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:55.520 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:55 smithi195 ceph-mon[31427]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:55.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[27560]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[27560]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[27560]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[39324]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[39324]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:55 smithi152 ceph-mon[39324]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:56.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:56 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:53:56.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:56 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:53:56.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:56 smithi195 ceph-mon[31427]: osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603] boot 2024-04-26T04:53:56.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:56 smithi195 ceph-mon[31427]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:53:56.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:56 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:56.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:56 smithi195 ceph-mon[31427]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[27560]: osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603] boot 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[27560]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[27560]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:53:56.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:53:56.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[39324]: osd.6 [v2:172.21.15.195:6816/751496603,v1:172.21.15.195:6817/751496603] boot 2024-04-26T04:53:56.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[39324]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:53:56.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:56.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:56 smithi152 ceph-mon[39324]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:53:57.714 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:53:57.714 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:53:57.715 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-04-26T04:53:57.715 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-04-26T04:53:57.715 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00855494 s, 1.2 GB/s 2024-04-26T04:53:57.715 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:58.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:58.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:58 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.311 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi195:vg_nvme/lv_1 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:58.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:58.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:58 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:53:58.561 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:59.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:59 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi195 2024-04-26T04:53:59.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:59 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi195 to 5696M 2024-04-26T04:53:59.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:53:59 smithi195 ceph-mon[31427]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:53:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:59 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi195 2024-04-26T04:53:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:59 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi195 to 5696M 2024-04-26T04:53:59.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:53:59 smithi152 ceph-mon[27560]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:53:59.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:59 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi195 2024-04-26T04:53:59.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:59 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi195 to 5696M 2024-04-26T04:53:59.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:53:59 smithi152 ceph-mon[39324]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:54:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:00 smithi195 ceph-mon[31427]: pgmap v156: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:00 smithi195 ceph-mon[31427]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:54:00.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:00 smithi152 ceph-mon[27560]: pgmap v156: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:00 smithi152 ceph-mon[27560]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:54:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:00 smithi152 ceph-mon[39324]: pgmap v156: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:00 smithi152 ceph-mon[39324]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:54:01.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:54:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:54:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:01 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:54:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:54:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:01 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:01.477 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:54:01.477 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:54:01.477 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:01 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:02.180 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:02 smithi195 ceph-mon[31427]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:02.180 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:02 smithi195 ceph-mon[31427]: pgmap v158: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:02 smithi152 ceph-mon[27560]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:02 smithi152 ceph-mon[27560]: pgmap v158: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:02 smithi152 ceph-mon[39324]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:02 smithi152 ceph-mon[39324]: pgmap v158: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/619048760' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]: dispatch 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]: dispatch 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[27560]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]': finished 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[27560]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/619048760' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]: dispatch 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]: dispatch 2024-04-26T04:54:03.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[39324]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]': finished 2024-04-26T04:54:03.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[39324]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:54:03.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:03 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:03.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:03 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/619048760' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]: dispatch 2024-04-26T04:54:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:03 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]: dispatch 2024-04-26T04:54:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:03 smithi195 ceph-mon[31427]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4449492c-2bd2-46c1-96ca-7f929946e4fe"}]': finished 2024-04-26T04:54:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:03 smithi195 ceph-mon[31427]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:54:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:03 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:04.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:04 smithi152 ceph-mon[27560]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:54:04.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:04 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/2827561075' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:54:04.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:04 smithi152 ceph-mon[39324]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:54:04.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:04 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/2827561075' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:54:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:04 smithi195 ceph-mon[31427]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:54:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:04 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/2827561075' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:54:06.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:06 smithi195 ceph-mon[31427]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-04-26T04:54:06.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:06 smithi152 ceph-mon[27560]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-04-26T04:54:06.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:06 smithi152 ceph-mon[39324]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-04-26T04:54:08.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:08 smithi195 ceph-mon[31427]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:54:08.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:08 smithi152 ceph-mon[27560]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:54:08.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:08 smithi152 ceph-mon[39324]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:54:10.362 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:10 smithi195 ceph-mon[31427]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-26T04:54:10.362 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:54:10.362 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:10 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:10.362 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:10 smithi195 ceph-mon[31427]: Deploying daemon osd.7 on smithi195 2024-04-26T04:54:10.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[27560]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-26T04:54:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:54:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[27560]: Deploying daemon osd.7 on smithi195 2024-04-26T04:54:10.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[39324]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-26T04:54:10.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:54:10.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:10.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:10 smithi152 ceph-mon[39324]: Deploying daemon osd.7 on smithi195 2024-04-26T04:54:12.263 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:12 smithi195 ceph-mon[31427]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:54:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:12 smithi152 ceph-mon[27560]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:54:12.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:12 smithi152 ceph-mon[39324]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:54:14.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:14 smithi195 ceph-mon[31427]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:14.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:14 smithi152 ceph-mon[27560]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:14.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:14 smithi152 ceph-mon[39324]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:15.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:15 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:15.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:15 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:15.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:15 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:15.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:15 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:15.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:15 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:15.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:15 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:15.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:15 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:15.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:15 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:15.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:15 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:16 smithi195 ceph-mon[31427]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:16.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:16 smithi152 ceph-mon[27560]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:16.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:16 smithi152 ceph-mon[39324]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:17.381 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 7 on host 'smithi195' 2024-04-26T04:54:17.985 DEBUG:teuthology.orchestra.run.smithi195:osd.7> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.7.service 2024-04-26T04:54:17.988 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-26T04:54:17.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T04:54:18.252 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:18 smithi195 ceph-mon[31427]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:18 smithi195 ceph-mon[31427]: from='osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:54:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:18 smithi195 ceph-mon[31427]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:54:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:18 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:18 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:18 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:18 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.374 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[27560]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[27560]: from='osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[27560]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[39324]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[39324]: from='osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[39324]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.375 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.376 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:18.376 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:18 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[27560]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:54:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[27560]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:54:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[27560]: from='osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:54:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[27560]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:54:19.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[39324]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:54:19.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[39324]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:54:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[39324]: from='osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:54:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:19 smithi152 ceph-mon[39324]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:54:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:19 smithi195 ceph-mon[31427]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:54:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:19 smithi195 ceph-mon[31427]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:54:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:19 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:19 smithi195 ceph-mon[31427]: from='osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:54:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:19 smithi195 ceph-mon[31427]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-04-26T04:54:19.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 04:54:19 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T04:54:19.188+0000 7fa0a1874700 -1 osd.7 0 waiting for initial osdmap 2024-04-26T04:54:19.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 04:54:19 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T04:54:19.195+0000 7fa09c66f700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:54:20.253 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:54:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:20 smithi195 ceph-mon[31427]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:20 smithi195 ceph-mon[31427]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:54:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:20 smithi195 ceph-mon[31427]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:54:20.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:20 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:20.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:20 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:20.602 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[27560]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:20.602 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[27560]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:54:20.602 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[27560]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:54:20.603 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:20.603 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:20.603 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[39324]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:54:20.603 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[39324]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-04-26T04:54:20.603 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[39324]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:54:20.603 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:20.603 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:20 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:20.920 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1714107260,"num_in_osds":8,"osd_in_since":1714107242,"num_remapped_pgs":1} 2024-04-26T04:54:20.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:54:21.175 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:21.198 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: purged_snaps scrub starts 2024-04-26T04:54:21.198 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: purged_snaps scrub ok 2024-04-26T04:54:21.198 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800] boot 2024-04-26T04:54:21.198 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:54:21.198 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:21.198 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1039114980' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:54:21.466 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:21.466 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[27560]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:54:21.466 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: purged_snaps scrub starts 2024-04-26T04:54:21.466 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: purged_snaps scrub ok 2024-04-26T04:54:21.466 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800] boot 2024-04-26T04:54:21.467 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:54:21.467 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:21.467 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1039114980' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:54:21.467 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:21.467 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:21 smithi152 ceph-mon[39324]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:54:21.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: purged_snaps scrub starts 2024-04-26T04:54:21.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: purged_snaps scrub ok 2024-04-26T04:54:21.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: osd.7 [v2:172.21.15.195:6824/4077645800,v1:172.21.15.195:6825/4077645800] boot 2024-04-26T04:54:21.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:54:21.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:21.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1039114980' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:54:21.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:21.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:21 smithi195 ceph-mon[31427]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: Detected new or changed devices on smithi195 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi195 to 4272M 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:23.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:23.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:23.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:23.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:23.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:22 smithi195 ceph-mon[31427]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:54:23.087 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:54:23.088 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":58,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T04:54:23.056070+0000","last_up_change":"2024-04-26T04:54:20.182998+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6803","nonce":354037778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6805","nonce":354037778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6809","nonce":354037778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6807","nonce":354037778}]},"public_addr":"172.21.15.152:6803/354037778","cluster_addr":"172.21.15.152:6805/354037778","heartbeat_back_addr":"172.21.15.152:6809/354037778","heartbeat_front_addr":"172.21.15.152:6807/354037778","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6811","nonce":1928910910}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6813","nonce":1928910910}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6817","nonce":1928910910}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6815","nonce":1928910910}]},"public_addr":"172.21.15.152:6811/1928910910","cluster_addr":"172.21.15.152:6813/1928910910","heartbeat_back_addr":"172.21.15.152:6817/1928910910","heartbeat_front_addr":"172.21.15.152:6815/1928910910","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6819","nonce":2274658721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6821","nonce":2274658721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6825","nonce":2274658721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6823","nonce":2274658721}]},"public_addr":"172.21.15.152:6819/2274658721","cluster_addr":"172.21.15.152:6821/2274658721","heartbeat_back_addr":"172.21.15.152:6825/2274658721","heartbeat_front_addr":"172.21.15.152:6823/2274658721","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6827","nonce":816595688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6829","nonce":816595688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6833","nonce":816595688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6831","nonce":816595688}]},"public_addr":"172.21.15.152:6827/816595688","cluster_addr":"172.21.15.152:6829/816595688","heartbeat_back_addr":"172.21.15.152:6833/816595688","heartbeat_front_addr":"172.21.15.152:6831/816595688","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:54:23.088 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: Detected new or changed devices on smithi195 2024-04-26T04:54:23.088 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.088 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.088 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi195 to 4272M 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.089 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[27560]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: Detected new or changed devices on smithi195 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi195 to 4272M 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.090 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:23.091 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:23.091 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:23.091 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:23.091 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:22 smithi152 ceph-mon[39324]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:54:23.912 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-26T04:52:21.033317+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-26T04:54:23.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-26T04:54:24.164 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:24.287 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:24 smithi152 ceph-mon[27560]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:24.287 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:24 smithi152 ceph-mon[27560]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:54:24.287 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:24 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1029645281' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:24.287 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:24 smithi152 ceph-mon[39324]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:24.287 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:24 smithi152 ceph-mon[39324]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:54:24.287 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:24 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1029645281' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:24.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:24 smithi195 ceph-mon[31427]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:24.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:24 smithi195 ceph-mon[31427]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:54:24.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:24 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1029645281' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:25.189 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:25 smithi152 ceph-mon[27560]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:54:25.190 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:25 smithi152 ceph-mon[39324]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:54:25.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:25 smithi195 ceph-mon[31427]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:54:25.895 INFO:teuthology.orchestra.run.smithi152.stdout:pg_num: 1 2024-04-26T04:54:26.212 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:26 smithi152 ceph-mon[27560]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:26.212 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:26 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1351342428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:54:26.212 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:26 smithi152 ceph-mon[39324]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:26.213 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:26 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1351342428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:54:26.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:26 smithi195 ceph-mon[31427]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:26.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:26 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1351342428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:54:26.556 INFO:tasks.cephadm:Adding prometheus.a on smithi195 2024-04-26T04:54:26.556 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi195=a' 2024-04-26T04:54:26.818 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:54:28.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:28 smithi152 ceph-mon[27560]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-26T04:54:28.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:28 smithi152 ceph-mon[39324]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-26T04:54:28.384 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:28 smithi195 ceph-mon[31427]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-26T04:54:28.696 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled prometheus update... 2024-04-26T04:54:29.319 DEBUG:teuthology.orchestra.run.smithi195:prometheus.a> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-26T04:54:29.322 INFO:tasks.cephadm:Adding node-exporter.a on smithi152 2024-04-26T04:54:29.322 INFO:tasks.cephadm:Adding node-exporter.b on smithi195 2024-04-26T04:54:29.322 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi152=a;smithi195=b' 2024-04-26T04:54:29.636 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:54:30.007 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:30.007 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: Saving service prometheus spec with placement smithi195=a;count:1 2024-04-26T04:54:30.007 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:30.007 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:30.008 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:30.008 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:30.008 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:30.008 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: Deploying daemon prometheus.a on smithi195 2024-04-26T04:54:30.008 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:29 smithi195 ceph-mon[31427]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T04:54:30.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:30.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: Saving service prometheus spec with placement smithi195=a;count:1 2024-04-26T04:54:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: Deploying daemon prometheus.a on smithi195 2024-04-26T04:54:30.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[39324]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T04:54:30.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:30.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: Saving service prometheus spec with placement smithi195=a;count:1 2024-04-26T04:54:30.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:30.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:30.133 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:30.133 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:30.133 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:30.133 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: Deploying daemon prometheus.a on smithi195 2024-04-26T04:54:30.133 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:29 smithi152 ceph-mon[27560]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T04:54:32.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:31 smithi195 ceph-mon[31427]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:54:32.076 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled node-exporter update... 2024-04-26T04:54:32.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:31 smithi152 ceph-mon[27560]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:54:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:31 smithi152 ceph-mon[39324]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:54:32.995 DEBUG:teuthology.orchestra.run.smithi152:node-exporter.a> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-26T04:54:32.998 DEBUG:teuthology.orchestra.run.smithi195:node-exporter.b> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-26T04:54:33.000 INFO:tasks.cephadm:Adding alertmanager.a on smithi152 2024-04-26T04:54:33.000 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi152=a' 2024-04-26T04:54:33.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:33 smithi195 ceph-mon[31427]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi152=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:33.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:33 smithi195 ceph-mon[31427]: Saving service node-exporter spec with placement smithi152=a;smithi195=b;count:2 2024-04-26T04:54:33.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:33 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:33.289 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:54:33.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:33 smithi152 ceph-mon[39324]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi152=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:33.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:33 smithi152 ceph-mon[39324]: Saving service node-exporter spec with placement smithi152=a;smithi195=b;count:2 2024-04-26T04:54:33.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:33 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:33 smithi152 ceph-mon[27560]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi152=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:33 smithi152 ceph-mon[27560]: Saving service node-exporter spec with placement smithi152=a;smithi195=b;count:2 2024-04-26T04:54:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:33 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:34 smithi152 ceph-mon[27560]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-26T04:54:34.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:34 smithi152 ceph-mon[39324]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-26T04:54:34.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:34 smithi195 ceph-mon[31427]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-26T04:54:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:36 smithi152 ceph-mon[27560]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-26T04:54:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:36 smithi152 ceph-mon[39324]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-26T04:54:36.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:36 smithi195 ceph-mon[31427]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-26T04:54:38.245 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:38 smithi195 ceph-mon[31427]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:54:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:38 smithi152 ceph-mon[27560]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:54:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:38 smithi152 ceph-mon[39324]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:54:38.401 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled alertmanager update... 2024-04-26T04:54:39.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:39 smithi195 ceph-mon[31427]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:39.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:39 smithi195 ceph-mon[31427]: Saving service alertmanager spec with placement smithi152=a;count:1 2024-04-26T04:54:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:39 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:39 smithi195 ceph-mon[31427]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:39.837 DEBUG:teuthology.orchestra.run.smithi152:alertmanager.a> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-26T04:54:39.840 INFO:tasks.cephadm:Adding grafana.a on smithi195 2024-04-26T04:54:39.840 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi195=a' 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[27560]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[27560]: Saving service alertmanager spec with placement smithi152=a;count:1 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[27560]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[39324]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[39324]: Saving service alertmanager spec with placement smithi152=a;count:1 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:39.865 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:39 smithi152 ceph-mon[39324]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:41.140 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:54:42.085 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:41 smithi195 ceph-mon[31427]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:42.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:41 smithi152 ceph-mon[27560]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:42.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:41 smithi152 ceph-mon[39324]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:43.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:54:43 smithi195 systemd[1]: Started Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:54:44.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:43 smithi195 ceph-mon[31427]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:44.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:43 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:54:44.128 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled grafana update... 2024-04-26T04:54:44.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[27560]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:44.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:54:44.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[39324]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:44.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:44.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:43 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:54:44.740 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:44 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ignoring --setuser ceph since I am not root 2024-04-26T04:54:44.740 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:44 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ignoring --setgroup ceph since I am not root 2024-04-26T04:54:44.740 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:44 smithi195 ceph-mgr[33539]: -- 172.21.15.195:0/3034797201 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561a91b4c9c0 con 0x561a91b22800 2024-04-26T04:54:44.765 DEBUG:teuthology.orchestra.run.smithi195:grafana.a> sudo journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service 2024-04-26T04:54:44.767 INFO:tasks.cephadm:Setting up client nodes... 2024-04-26T04:54:44.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:54:44.972 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setuser ceph since I am not root 2024-04-26T04:54:44.973 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setgroup ceph since I am not root 2024-04-26T04:54:44.973 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:44 smithi152 ceph-mgr[27842]: -- 172.21.15.152:0/3531479264 <== mon.2 v2:172.21.15.152:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d7d8f4e9c0 con 0x55d7d8f25000 2024-04-26T04:54:44.973 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:44.802+0000 7fea0dc301c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:54:44.973 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:44.971+0000 7fea0dc301c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:54:45.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:44 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:44.800+0000 7f8c34f4d1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:54:45.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:44 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:44.959+0000 7f8c34f4d1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:54:45.034 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:45.289 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[27560]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:45.289 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[27560]: Saving service grafana spec with placement smithi195=a;count:1 2024-04-26T04:54:45.289 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:45.289 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[27560]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:54:45.289 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[27560]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:54:45.290 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:45.090+0000 7fea0dc301c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:54:45.290 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[39324]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:45.290 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[39324]: Saving service grafana spec with placement smithi195=a;count:1 2024-04-26T04:54:45.290 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:45.290 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[39324]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:54:45.290 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-mon[39324]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:54:45.362 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:45.058+0000 7f8c34f4d1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:54:45.363 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-mon[31427]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:45.363 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-mon[31427]: Saving service grafana spec with placement smithi195=a;count:1 2024-04-26T04:54:45.363 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' 2024-04-26T04:54:45.363 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-mon[31427]: from='mgr.14148 172.21.15.152:0/1437510584' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:54:45.363 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-mon[31427]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:54:45.540 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:45.407+0000 7fea0dc301c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:54:45.540 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:45 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:45.496+0000 7fea0dc301c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:54:45.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:45.361+0000 7f8c34f4d1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:54:45.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:45 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:45.443+0000 7f8c34f4d1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:54:46.855 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:46.666+0000 7fea0dc301c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:54:46.855 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:46.837+0000 7fea0dc301c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:54:46.911 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:46.578+0000 7f8c34f4d1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:54:46.911 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:46.741+0000 7f8c34f4d1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:54:46.911 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:46.910+0000 7f8c34f4d1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:54:47.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:47.014+0000 7fea0dc301c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:54:47.161 INFO:teuthology.orchestra.run.smithi152.stdout:[client.0] 2024-04-26T04:54:47.161 INFO:teuthology.orchestra.run.smithi152.stdout: key = AQCXMytmx3piCRAADY73/4SGR7zyLkowpuXfCw== 2024-04-26T04:54:47.447 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2628145393' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:47.447 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-mon[27560]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:47.447 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-mon[27560]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:54:47.447 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2628145393' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:47.447 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-mon[39324]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:47.448 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-mon[39324]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:54:47.448 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:47.286+0000 7fea0dc301c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:54:47.448 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:47.377+0000 7fea0dc301c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:54:47.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:47 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2628145393' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:47.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:47 smithi195 ceph-mon[31427]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:47.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:47 smithi195 ceph-mon[31427]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:54:47.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:47 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:47.170+0000 7f8c34f4d1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:54:47.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:47 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:47.260+0000 7f8c34f4d1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:54:47.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:47 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:47.351+0000 7f8c34f4d1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:54:47.759 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:47.478+0000 7fea0dc301c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:54:47.909 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:54:47.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-26T04:54:47.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-26T04:54:47.945 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:54:47.966 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:47 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:47.628+0000 7f8c34f4d1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:54:48.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:47.758+0000 7fea0dc301c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:54:48.210 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:54:48.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:48.092+0000 7f8c34f4d1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:54:48.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:48 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:48.229+0000 7fea0dc301c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:54:48.932 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:48.657+0000 7f8c34f4d1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:54:48.932 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:48.752+0000 7f8c34f4d1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:54:48.932 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:48.931+0000 7f8c34f4d1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:54:49.029 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:48 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:48.756+0000 7fea0dc301c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:54:49.029 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:48 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:48.850+0000 7fea0dc301c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:54:49.318 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:49.028+0000 7fea0dc301c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:54:49.496 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:49.224+0000 7f8c34f4d1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:54:49.496 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:49.494+0000 7f8c34f4d1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:54:49.575 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:49.317+0000 7fea0dc301c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:54:49.575 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:49.574+0000 7fea0dc301c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:54:49.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:49.738+0000 7f8c34f4d1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:54:49.879 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:49.815+0000 7fea0dc301c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:54:50.103 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:49.890+0000 7f8c34f4d1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:54:50.103 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:49.985+0000 7f8c34f4d1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:54:50.148 INFO:teuthology.orchestra.run.smithi195.stdout:[client.1] 2024-04-26T04:54:50.148 INFO:teuthology.orchestra.run.smithi195.stdout: key = AQCaMytmHSqOCBAA/+jrHh46qU9+3G8UmNnKew== 2024-04-26T04:54:50.256 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-mon[27560]: from='client.? 172.21.15.195:0/3069701509' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:50.256 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-mon[27560]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:50.256 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-mon[27560]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:54:50.257 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-mon[39324]: from='client.? 172.21.15.195:0/3069701509' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:50.257 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-mon[39324]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:50.257 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-mon[39324]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:54:50.257 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:49.949+0000 7fea0dc301c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:54:50.257 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:50.044+0000 7fea0dc301c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:54:50.257 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:50.162+0000 7fea0dc301c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:54:50.257 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:50 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:50.255+0000 7fea0dc301c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:54:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:50 smithi195 ceph-mon[31427]: from='client.? 172.21.15.195:0/3069701509' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:50 smithi195 ceph-mon[31427]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:54:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:50 smithi195 ceph-mon[31427]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:54:50.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:50.101+0000 7f8c34f4d1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:54:50.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:50.201+0000 7f8c34f4d1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:54:50.829 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-04-26T04:54:50.829 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-26T04:54:50.829 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-26T04:54:50.905 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-26T04:54:50.905 DEBUG:teuthology.orchestra.run.smithi152:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-26T04:54:51.179 INFO:teuthology.misc.health.smithi152.stdout: 2024-04-26T04:54:51.179 INFO:teuthology.misc.health.smithi152.stdout:{"epoch":59,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T04:54:24.057276+0000","last_up_change":"2024-04-26T04:54:20.182998+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6803","nonce":354037778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6805","nonce":354037778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6809","nonce":354037778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6807","nonce":354037778}]},"public_addr":"172.21.15.152:6803/354037778","cluster_addr":"172.21.15.152:6805/354037778","heartbeat_back_addr":"172.21.15.152:6809/354037778","heartbeat_front_addr":"172.21.15.152:6807/354037778","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6811","nonce":1928910910}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6813","nonce":1928910910}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6817","nonce":1928910910}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6815","nonce":1928910910}]},"public_addr":"172.21.15.152:6811/1928910910","cluster_addr":"172.21.15.152:6813/1928910910","heartbeat_back_addr":"172.21.15.152:6817/1928910910","heartbeat_front_addr":"172.21.15.152:6815/1928910910","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6819","nonce":2274658721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6821","nonce":2274658721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6825","nonce":2274658721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6823","nonce":2274658721}]},"public_addr":"172.21.15.152:6819/2274658721","cluster_addr":"172.21.15.152:6821/2274658721","heartbeat_back_addr":"172.21.15.152:6825/2274658721","heartbeat_front_addr":"172.21.15.152:6823/2274658721","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6827","nonce":816595688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6829","nonce":816595688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6833","nonce":816595688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6831","nonce":816595688}]},"public_addr":"172.21.15.152:6827/816595688","cluster_addr":"172.21.15.152:6829/816595688","heartbeat_back_addr":"172.21.15.152:6833/816595688","heartbeat_front_addr":"172.21.15.152:6831/816595688","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:54:51.191 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-26T04:54:51.191 INFO:tasks.cephadm:Creating RBD pool 2024-04-26T04:54:51.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-26T04:54:51.196 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:51.012+0000 7f8c34f4d1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:54:51.330 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/674929312' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:51.331 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:51.065+0000 7fea0dc301c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:54:51.331 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:51.268+0000 7fea0dc301c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:54:51.331 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/674929312' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/674929312' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:51.204+0000 7f8c34f4d1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: [26/Apr/2024:04:54:51] ENGINE Bus STARTING 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: CherryPy Checker: 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: The Application mounted at '' has an empty config. 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: [26/Apr/2024:04:54:51] ENGINE Serving on http://:::9283 2024-04-26T04:54:51.449 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:54:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: [26/Apr/2024:04:54:51] ENGINE Bus STARTED 2024-04-26T04:54:51.471 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:51.594 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:54:51] ENGINE Bus STARTING 2024-04-26T04:54:51.594 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: CherryPy Checker: 2024-04-26T04:54:51.594 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: The Application mounted at '' has an empty config. 2024-04-26T04:54:51.594 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:54:51.594 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:54:51] ENGINE Serving on http://:::9283 2024-04-26T04:54:51.594 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:54:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:54:51] ENGINE Bus STARTED 2024-04-26T04:54:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: Standby manager daemon x restarted 2024-04-26T04:54:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: Standby manager daemon x started 2024-04-26T04:54:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:54:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:54:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:54:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: Active manager daemon y restarted 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: Activating manager daemon y 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: mgrmap e17: y(active, starting, since 0.00766482s), standbys: x 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:54:52.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: Manager daemon y is now available 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:54:52.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:52 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:54:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: Standby manager daemon x restarted 2024-04-26T04:54:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: Standby manager daemon x started 2024-04-26T04:54:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:54:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: Active manager daemon y restarted 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: Activating manager daemon y 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: mgrmap e17: y(active, starting, since 0.00766482s), standbys: x 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:54:52.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: Manager daemon y is now available 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:52.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: Standby manager daemon x restarted 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: Standby manager daemon x started 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/3894133969' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: Active manager daemon y restarted 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: Activating manager daemon y 2024-04-26T04:54:52.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: mgrmap e17: y(active, starting, since 0.00766482s), standbys: x 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:54:52.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: Manager daemon y is now available 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:52.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:54:52.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:52 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:54:53.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:53 smithi195 ceph-mon[31427]: mgrmap e18: y(active, since 1.01078s), standbys: x 2024-04-26T04:54:53.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:53 smithi195 ceph-mon[31427]: [26/Apr/2024:04:54:52] ENGINE Bus STARTING 2024-04-26T04:54:53.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:53 smithi195 ceph-mon[31427]: [26/Apr/2024:04:54:52] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:54:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:53 smithi195 ceph-mon[31427]: [26/Apr/2024:04:54:52] ENGINE Bus STARTED 2024-04-26T04:54:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:53 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[27560]: mgrmap e18: y(active, since 1.01078s), standbys: x 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[27560]: [26/Apr/2024:04:54:52] ENGINE Bus STARTING 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[27560]: [26/Apr/2024:04:54:52] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[27560]: [26/Apr/2024:04:54:52] ENGINE Bus STARTED 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[39324]: mgrmap e18: y(active, since 1.01078s), standbys: x 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[39324]: [26/Apr/2024:04:54:52] ENGINE Bus STARTING 2024-04-26T04:54:53.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[39324]: [26/Apr/2024:04:54:52] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:54:53.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[39324]: [26/Apr/2024:04:54:52] ENGINE Bus STARTED 2024-04-26T04:54:53.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:53 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.293 INFO:teuthology.orchestra.run.smithi152.stderr:pool 'rbd' created 2024-04-26T04:54:54.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[39324]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:54.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2417466819' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:54:54.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[39324]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:54:54.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[27560]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2417466819' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[27560]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.613 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:54 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:54 smithi195 ceph-mon[31427]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:54 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2417466819' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:54:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:54 smithi195 ceph-mon[31427]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:54:54.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:54 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:54 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:54 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:54.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:54 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-26T04:54:55.290 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2417466819' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:54:55.290 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:54:55.290 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.290 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.290 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.290 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.325 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:55.588 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.588 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.588 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi195 to 4016M 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2417466819' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:54:55.589 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi195 to 4016M 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.590 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.591 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.591 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:55.591 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:55 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:55.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2417466819' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:54:55.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:54:55.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi195 to 4016M 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:55 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:56.610 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:54:56.610 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:54:56.610 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: pgmap v6: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:56.610 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:54:56.610 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:54:56.610 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:54:56.611 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: pgmap v6: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.612 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: pgmap v6: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:56.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:54:57.383 INFO:teuthology.orchestra.run.smithi152.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-26T04:54:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:54:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:54:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[39324]: Deploying daemon alertmanager.a on smithi152 2024-04-26T04:54:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[39324]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:54:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1959476377' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[39324]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:57.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:54:57.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:54:57.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[27560]: Deploying daemon alertmanager.a on smithi152 2024-04-26T04:54:57.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[27560]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:54:57.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1959476377' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:57.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:57 smithi152 ceph-mon[27560]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:57.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:57 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:54:57.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:57 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:54:57.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:57 smithi195 ceph-mon[31427]: Deploying daemon alertmanager.a on smithi152 2024-04-26T04:54:57.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:57 smithi195 ceph-mon[31427]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:54:57.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:57 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1959476377' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:57.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:57 smithi195 ceph-mon[31427]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:58.588 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:54:58.588 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:54:58.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:54:58.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:58 smithi152 ceph-mon[27560]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:58.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:58 smithi152 ceph-mon[27560]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:58.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:58 smithi152 ceph-mon[27560]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:54:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:58 smithi152 ceph-mon[39324]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:58 smithi152 ceph-mon[39324]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:58 smithi152 ceph-mon[39324]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:54:58.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:58 smithi195 ceph-mon[31427]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:58.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:58 smithi195 ceph-mon[31427]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:58.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:58 smithi195 ceph-mon[31427]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:54:58.849 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:59.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:59 smithi195 ceph-mon[31427]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:59.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:54:59 smithi195 ceph-mon[31427]: Cluster is now healthy 2024-04-26T04:54:59.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:59 smithi152 ceph-mon[39324]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:59.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:54:59 smithi152 ceph-mon[39324]: Cluster is now healthy 2024-04-26T04:54:59.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:59 smithi152 ceph-mon[27560]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:59.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:54:59 smithi152 ceph-mon[27560]: Cluster is now healthy 2024-04-26T04:55:00.685 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:00 smithi195 ceph-mon[31427]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-26T04:55:00.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:00 smithi152 ceph-mon[27560]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-26T04:55:00.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:00 smithi152 ceph-mon[39324]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-26T04:55:01.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:55:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:55:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:02.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:02 smithi152 ceph-mon[27560]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:55:02.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:02 smithi152 ceph-mon[39324]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:55:02.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:02 smithi195 ceph-mon[31427]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:55:02.806 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:55:03.505 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":20,"flags":0,"active_gid":14430,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1036587324},{"type":"v1","addr":"172.21.15.152:6801","nonce":1036587324}]},"active_addr":"172.21.15.152:6801/1036587324","active_change":"2024-04-26T04:54:51.271327+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.152:8443/","prometheus":"http://172.21.15.152:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3664778945}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3677441985}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":1107620726}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3193673978}]}]} 2024-04-26T04:55:03.509 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:55:03.509 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:55:03.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:55:03.756 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:03 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/572201679' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:55:03.757 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:03 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/572201679' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:55:03.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:03 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/572201679' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:55:04.727 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:04 smithi152 ceph-mon[39324]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:55:04.728 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:04 smithi152 ceph-mon[27560]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:55:04.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:04 smithi195 ceph-mon[31427]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:55:05.050 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:06.297 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:06 smithi152 systemd[1]: Starting Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:55:06.571 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:06 smithi152 ceph-mon[27560]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T04:55:06.571 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:06 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:06.571 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:06 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:06.571 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:06 smithi152 ceph-mon[39324]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T04:55:06.572 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:06 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:06.572 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:06 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:06 smithi195 ceph-mon[31427]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T04:55:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:06 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:06 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:07.296 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:55:07.296 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":64,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T04:54:57.370439+0000","last_up_change":"2024-04-26T04:54:20.182998+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:54:53.698027+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6803","nonce":354037778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6805","nonce":354037778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6809","nonce":354037778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6807","nonce":354037778}]},"public_addr":"172.21.15.152:6803/354037778","cluster_addr":"172.21.15.152:6805/354037778","heartbeat_back_addr":"172.21.15.152:6809/354037778","heartbeat_front_addr":"172.21.15.152:6807/354037778","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6811","nonce":1928910910}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6813","nonce":1928910910}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6817","nonce":1928910910}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6815","nonce":1928910910}]},"public_addr":"172.21.15.152:6811/1928910910","cluster_addr":"172.21.15.152:6813/1928910910","heartbeat_back_addr":"172.21.15.152:6817/1928910910","heartbeat_front_addr":"172.21.15.152:6815/1928910910","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6819","nonce":2274658721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6821","nonce":2274658721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6825","nonce":2274658721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6823","nonce":2274658721}]},"public_addr":"172.21.15.152:6819/2274658721","cluster_addr":"172.21.15.152:6821/2274658721","heartbeat_back_addr":"172.21.15.152:6825/2274658721","heartbeat_front_addr":"172.21.15.152:6823/2274658721","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6827","nonce":816595688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6829","nonce":816595688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6833","nonce":816595688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6831","nonce":816595688}]},"public_addr":"172.21.15.152:6827/816595688","cluster_addr":"172.21.15.152:6829/816595688","heartbeat_back_addr":"172.21.15.152:6833/816595688","heartbeat_front_addr":"172.21.15.152:6831/816595688","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:6800/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/604484318":"2024-04-27T04:54:51.271123+0000","172.21.15.152:6801/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/2636634813":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1523725454":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2161488710":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:55:07.380 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 podman[70891]: 2024-04-26 04:55:07.234525242 +0000 UTC m=+0.021634241 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:55:07.380 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 podman[70891]: 2024-04-26 04:55:07.333751563 +0000 UTC m=+0.120860549 volume create c049d311c606be0a31e1dd4d4a563fc42ace760695e36ce17d428bae807812cb 2024-04-26T04:55:07.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:07 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/138479489' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:07.789 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/138479489' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:07.789 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/138479489' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:07.789 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 podman[70891]: 2024-04-26 04:55:07.426081612 +0000 UTC m=+0.213190601 container create 1e69262794a8e083b883d8759d5fb737d5bdfd4f0c8a635ba61b052d1170cf6f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:08.244 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 podman[70891]: 2024-04-26 04:55:07.904037039 +0000 UTC m=+0.691146040 container init 1e69262794a8e083b883d8759d5fb737d5bdfd4f0c8a635ba61b052d1170cf6f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:08.244 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 podman[70891]: 2024-04-26 04:55:07.909634642 +0000 UTC m=+0.696743628 container start 1e69262794a8e083b883d8759d5fb737d5bdfd4f0c8a635ba61b052d1170cf6f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.921Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.921Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.922Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.152 port=9094 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.923Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.946Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.947Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.949Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:07.949Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:08 smithi152 bash[70891]: 1e69262794a8e083b883d8759d5fb737d5bdfd4f0c8a635ba61b052d1170cf6f 2024-04-26T04:55:08.245 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:08 smithi152 systemd[1]: Started Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:55:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:55:08] ENGINE Bus STOPPING 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:55:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:55:08] ENGINE Bus STOPPED 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:55:08] ENGINE Bus STARTING 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:55:08] ENGINE Serving on http://:::9283 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:55:08] ENGINE Bus STARTED 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:55:08.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.775 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:55:08.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:55:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:09.039 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:09.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:09 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:55:09.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:09 smithi195 ceph-mon[31427]: Deploying daemon grafana.a on smithi195 2024-04-26T04:55:09.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:09 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:55:09.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:09 smithi152 ceph-mon[27560]: Deploying daemon grafana.a on smithi195 2024-04-26T04:55:09.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:09 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:55:09.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:09 smithi152 ceph-mon[39324]: Deploying daemon grafana.a on smithi195 2024-04-26T04:55:10.323 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:09.923Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000024179s 2024-04-26T04:55:10.614 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:10 smithi152 ceph-mon[27560]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:10.614 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:10 smithi152 ceph-mon[39324]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:10 smithi195 ceph-mon[31427]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:10.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:55:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:55:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:11.038 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:55:11.038 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":64,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T04:54:57.370439+0000","last_up_change":"2024-04-26T04:54:20.182998+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:54:53.698027+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6803","nonce":354037778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6805","nonce":354037778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6809","nonce":354037778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6807","nonce":354037778}]},"public_addr":"172.21.15.152:6803/354037778","cluster_addr":"172.21.15.152:6805/354037778","heartbeat_back_addr":"172.21.15.152:6809/354037778","heartbeat_front_addr":"172.21.15.152:6807/354037778","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6811","nonce":1928910910}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6813","nonce":1928910910}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6817","nonce":1928910910}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6815","nonce":1928910910}]},"public_addr":"172.21.15.152:6811/1928910910","cluster_addr":"172.21.15.152:6813/1928910910","heartbeat_back_addr":"172.21.15.152:6817/1928910910","heartbeat_front_addr":"172.21.15.152:6815/1928910910","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6819","nonce":2274658721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6821","nonce":2274658721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6825","nonce":2274658721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6823","nonce":2274658721}]},"public_addr":"172.21.15.152:6819/2274658721","cluster_addr":"172.21.15.152:6821/2274658721","heartbeat_back_addr":"172.21.15.152:6825/2274658721","heartbeat_front_addr":"172.21.15.152:6823/2274658721","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6827","nonce":816595688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6829","nonce":816595688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6833","nonce":816595688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6831","nonce":816595688}]},"public_addr":"172.21.15.152:6827/816595688","cluster_addr":"172.21.15.152:6829/816595688","heartbeat_back_addr":"172.21.15.152:6833/816595688","heartbeat_front_addr":"172.21.15.152:6831/816595688","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:6800/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/604484318":"2024-04-27T04:54:51.271123+0000","172.21.15.152:6801/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/2636634813":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1523725454":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2161488710":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:55:11.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:11 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2169290629' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:11.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:11 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:11.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:11 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2169290629' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:11.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:11 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:11.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:11 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2169290629' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:11.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:11 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:11.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:55:11.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:55:11.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:55:11.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:55:11.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:55:11.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:55:11.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:55:11.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:55:12.277 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:12.638 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:12.638 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:12.638 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:12.639 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:12.639 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:12.641 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:12.767 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:12 smithi152 ceph-mon[27560]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:12.767 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:12 smithi152 ceph-mon[39324]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:12 smithi195 ceph-mon[31427]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:13.039 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:14.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:14 smithi152 ceph-mon[27560]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:14.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:14 smithi152 ceph-mon[39324]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:14 smithi195 ceph-mon[31427]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:16.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:16 smithi152 ceph-mon[27560]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:16.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:16 smithi152 ceph-mon[39324]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:16.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:16 smithi195 ceph-mon[31427]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:18.293 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:55:17 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:55:17.925Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001784667s 2024-04-26T04:55:18.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:18 smithi152 ceph-mon[27560]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:18.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:18 smithi152 ceph-mon[39324]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:18.638 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:18 smithi195 ceph-mon[31427]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:20.499 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:20 smithi195 ceph-mon[31427]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:20.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:20 smithi152 ceph-mon[27560]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:20.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:20 smithi152 ceph-mon[39324]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:20.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:55:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:55:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:21.193 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 podman[62167]: 2024-04-26 04:55:21.002455428 +0000 UTC m=+0.017129269 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:55:21.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 podman[62167]: 2024-04-26 04:55:21.319394342 +0000 UTC m=+0.334068183 container init 8dd84a1289295be629fbdc186255a3046b51f4c9ffd6c433a078d24ec90b3e1f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, vendor=Red Hat, Inc., name=ubi8, build-date=2022-04-29T11:48:11.055897, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 podman[62167]: 2024-04-26 04:55:21.32508152 +0000 UTC m=+0.339755362 container start 8dd84a1289295be629fbdc186255a3046b51f4c9ffd6c433a078d24ec90b3e1f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, version=8.5, maintainer=Paul Cuzner , name=ubi8, com.redhat.component=ubi8-container, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 bash[62167]: 8dd84a1289295be629fbdc186255a3046b51f4c9ffd6c433a078d24ec90b3e1f 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.478985827Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479189541Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.47921324Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479226162Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479238155Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T04:55:21.570 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479249621Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479260361Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479271113Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479281752Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479293932Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479304143Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.47931475Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.47932535Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=settings t=2024-04-26T04:55:21.479335837Z level=info msg="App mode production" 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore t=2024-04-26T04:55:21.479393788Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore t=2024-04-26T04:55:21.479417287Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.48026876Z level=info msg="Starting DB migrations" 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.480356309Z level=info msg="Executing migration" id="create migration_log table" 2024-04-26T04:55:21.571 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.480860346Z level=info msg="Executing migration" id="create user table" 2024-04-26T04:55:21.572 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.481269383Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-26T04:55:21.572 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.481722217Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-26T04:55:21.572 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.482191202Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-26T04:55:21.572 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.482656094Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.483040095Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.483675491Z level=info msg="Executing migration" id="create user table v2" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.484204535Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.484532331Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.48490153Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.48522455Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.485509678Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.485851932Z level=info msg="Executing migration" id="Update user table charset" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.486130812Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-26T04:55:21.573 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.486468198Z level=info msg="Executing migration" id="Add missing user data" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.486739848Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.487124345Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.487479366Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.487821754Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.48892666Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.489332146Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.489731275Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.490125801Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.490546004Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.490916648Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.491168714Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-26T04:55:21.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.491552829Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.491925549Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.492294713Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.492627394Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.493235093Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.49359197Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.493958826Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.494337569Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.494673034Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.495076193Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.495383348Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.495664886Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.49595203Z level=info msg="Executing migration" id="create star table" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.496287302Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.496685132Z level=info msg="Executing migration" id="create org table v1" 2024-04-26T04:55:21.575 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.497075367Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.49747186Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.497810033Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.498193113Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.498590711Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.498955126Z level=info msg="Executing migration" id="Update org table charset" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.499218841Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.499449879Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.499703996Z level=info msg="Executing migration" id="create dashboard table" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.500058099Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.500459178Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.50085579Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.501249994Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-26T04:55:21.576 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.50165757Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.502035539Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.502831192Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.503283619Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.503670583Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.504067524Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.50436708Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.504692557Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.504949919Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.505436573Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.505854876Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.50637241Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.506720233Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.50717936Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.507585333Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-26T04:55:21.577 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.507939098Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.508200612Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.508434384Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.508934321Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.509447206Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.509877552Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.510411708Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.510674154Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.511064776Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.511427738Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.511662662Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.512065924Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.512418443Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-26T04:55:21.578 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.513313826Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.513670925Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.514084474Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.514467734Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.514739879Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.51507672Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.515568956Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.51592369Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.516210032Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.51646023Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.51682959Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.517385186Z level=info msg="Executing migration" id="create data_source table" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.517772556Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.518201691Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-26T04:55:21.579 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.518587731Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.518923208Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.519348858Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.520385403Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.520769645Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.521158558Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.521554183Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.521843306Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.522362528Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.522818245Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.523089807Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.523370572Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.523822443Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.524114112Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-26T04:55:21.580 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.524389403Z level=info msg="Executing migration" id="Add uid column" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.524875598Z level=info msg="Executing migration" id="Update uid value" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.525169458Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.525528291Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.525966924Z level=info msg="Executing migration" id="create api_key table" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.526355613Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.526876309Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.527313599Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.52774913Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.528137132Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.528523851Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-26T04:55:21.581 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.528873983Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-26T04:55:21.582 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.529881912Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-26T04:55:21.582 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.53029414Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.530667531Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.531083211Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.531487283Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.531815953Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.532163854Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.532436342Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.532929354Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.533475794Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.533740387Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.534269721Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.534841144Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.535277862Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-26T04:55:21.583 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.535595546Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.535947704Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.536349851Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.536745383Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.537174146Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.537472405Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.537726644Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.538302436Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.538860998Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.539193524Z level=info msg="Executing migration" id="create quota table v1" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.539562778Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.539973367Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.54025938Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.540632036Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-26T04:55:21.584 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.541052997Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.541654497Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.541914957Z level=info msg="Executing migration" id="create session table" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.542334594Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.542589192Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.542841009Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.543292767Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.543680172Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.543934235Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.54421545Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.544467319Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.544724792Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.545111739Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.545388996Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-26T04:55:21.585 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.546000422Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.546331325Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.546867038Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.547465658Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.547721141Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.548134976Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.548573877Z level=info msg="Executing migration" id="create alert table v1" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.549021952Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.549466778Z level=info msg="Executing migration" id="add index alert state" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.549876651Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.550303343Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.55066771Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.551087276Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-26T04:55:21.586 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.551493503Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.552877053Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.553254803Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.553670529Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.553963269Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.554313455Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.554653578Z level=info msg="Executing migration" id="Add column is_default" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.555326185Z level=info msg="Executing migration" id="Add column frequency" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.555966014Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.556601016Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.557216802Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.557636181Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.557896022Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-26T04:55:21.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.55817969Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.558553445Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.558998562Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.55937518Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.559721423Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.560157662Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.56077459Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.561410223Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.561720246Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.562167592Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.562556719Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.563182447Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.563468576Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.563835637Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-26T04:55:21.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.564281208Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.564579304Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.565000591Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.565407609Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.565808324Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.566227566Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.566672997Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.567113575Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.567391268Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.568055323Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.568446182Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.569113296Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.569495901Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.569935269Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-26T04:55:21.589 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.570339597Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.571851011Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.572259624Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.572674877Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.572961661Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.573307911Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.573580608Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.574229038Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.574866817Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.575306836Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.575719711Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-26T04:55:21.590 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.576030426Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-26T04:55:21.734 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:21 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:21.734 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:21 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:22.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.576674162Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-26T04:55:22.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.577115724Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-26T04:55:22.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.57741177Z level=info msg="Executing migration" id="Move region to single row" 2024-04-26T04:55:22.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.577764995Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-26T04:55:22.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.578163151Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-26T04:55:22.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.578558291Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.578951282Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.579356341Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.579710628Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.5801338Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.580441741Z level=info msg="Executing migration" id="create test_data table" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.58083196Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.581222097Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.581621847Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.58203415Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.582328672Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.582608334Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-26T04:55:22.032 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.582867439Z level=info msg="Executing migration" id="create team table" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.583255507Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.583691494Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.584129674Z level=info msg="Executing migration" id="create team member table" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.584514038Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.584954167Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.585369065Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.585769557Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.586454444Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.587140324Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.587815959Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-26T04:55:22.034 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.588209871Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.588634464Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.589059574Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.589466273Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.589953945Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.590369871Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.590773437Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.591181253Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.591597322Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.59186935Z level=info msg="Executing migration" id="create tag table" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.592250358Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.592647145Z level=info msg="Executing migration" id="create login attempt table" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.593024118Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-26T04:55:22.035 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.593425409Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.593787498Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.595656831Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.596043718Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.596451241Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.596730771Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.597091315Z level=info msg="Executing migration" id="create user auth table" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.597479788Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.597875249Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.598172379Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.598847774Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.599589703Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.600340342Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.60110032Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-26T04:55:22.036 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.601484676Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.602208003Z level=info msg="Executing migration" id="create server_lock table" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.602551951Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.602938882Z level=info msg="Executing migration" id="create user auth token table" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.603322647Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.603692901Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.604077094Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.604446961Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.605164353Z level=info msg="Executing migration" id="create cache_data table" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.605532758Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.605919743Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.606321163Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-26T04:55:22.037 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.606696307Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.606929266Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.607193032Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.607638752Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.608027796Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.608412401Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.608637147Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.609013Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.609383509Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.609966431Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.610350322Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.611123831Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.611512716Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-26T04:55:22.038 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.611752839Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.612125163Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.612520986Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.612869113Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.613131839Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.613485302Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.613827144Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.614218243Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.614595083Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.615396152Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.615762657Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.616141743Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.619433984Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-26T04:55:22.039 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.622642379Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.623059601Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.623440686Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.624254918Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.62462939Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.625017814Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.625406754Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.625776555Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.626066639Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.626819525Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.627597673Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.628384856Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.628754582Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.629150402Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-26T04:55:22.040 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.62993389Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-26T04:55:22.041 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.63070662Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-26T04:55:22.041 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.631108104Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-26T04:55:22.041 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.631863338Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-26T04:55:22.042 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.632627654Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-26T04:55:22.042 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.632871633Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-26T04:55:22.042 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.633307613Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-26T04:55:22.042 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.633690852Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.634083375Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.634338967Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.635144121Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.635927907Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.636696939Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.637507096Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.638297422Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.638559036Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.638903497Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.639677449Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.639934235Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.640770274Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-26T04:55:22.043 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.641168394Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.641954153Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.642308613Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.642687634Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.64351223Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.643863404Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.644289908Z level=info msg="Executing migration" id="create alert_image table" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.644633186Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.6450189Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.645297132Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.645663047Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.6460341Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.646109711Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:55:22.044 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.646333262Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-26T04:55:22.045 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.646614856Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-26T04:55:22.045 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.646963461Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-26T04:55:22.045 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.647075521Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-26T04:55:22.045 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.647189238Z level=info msg="alerts found to migrate" alerts=0 2024-04-26T04:55:22.045 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.647537911Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-26T04:55:22.045 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.647944928Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-26T04:55:22.046 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.648338722Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-26T04:55:22.046 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.648683302Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-26T04:55:22.046 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.649082295Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.649492295Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.649706124Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.650032941Z level=info msg="Executing migration" id="create data_keys table" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.650408613Z level=info msg="Executing migration" id="create secrets table" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.650797748Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.654994161Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.655842956Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.65611449Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.660248213Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.664455343Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.664817689Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.665224712Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.665517487Z level=info msg="Executing migration" id="create permission table" 2024-04-26T04:55:22.047 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.665864769Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.666260456Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.666652586Z level=info msg="Executing migration" id="create role table" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.667008533Z level=info msg="Executing migration" id="add column display_name" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.667886788Z level=info msg="Executing migration" id="add column group_name" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.668746397Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.669148287Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.669533927Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.669924188Z level=info msg="Executing migration" id="create team role table" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.670271972Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.670653771Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.671118077Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.671506955Z level=info msg="Executing migration" id="create user role table" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.671844977Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-26T04:55:22.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.672241102Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.672626729Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.673016227Z level=info msg="Executing migration" id="create builtin role table" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.673387829Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.673770214Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.674169013Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.675077441Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.675503471Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.675898026Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.676297137Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.676650384Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.676996125Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.677408847Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-26T04:55:22.049 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.678382733Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.678749976Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.679152822Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.679401334Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.679723639Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.68012818Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.680448932Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.680682593Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.681009456Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.681385009Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.68177018Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.68267692Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.682955459Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.683329968Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-26T04:55:22.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.683712427Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.684106589Z level=info msg="Executing migration" id="add correlation config column" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.685039146Z level=info msg="Executing migration" id="create entity_events table" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.685421479Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.68581433Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.685898678Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.686149127Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.686225271Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.686452969Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.686813745Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.687211408Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.687605472Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.688152233Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.688556427Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.688929218Z level=info msg="Executing migration" id="Drop public config table" 2024-04-26T04:55:22.051 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.689329053Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.68970235Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.690118936Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.690552916Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.690961734Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.693933502Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.694896229Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.695846305Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.696149851Z level=info msg="Executing migration" id="add share column" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.697108504Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.69750679Z level=info msg="Executing migration" id="create file table" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.697952282Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.698390599Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.698850026Z level=info msg="Executing migration" id="create file_meta table" 2024-04-26T04:55:22.052 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.699220629Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.699657472Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.699904472Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.700248028Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.700504234Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.700863786Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.701878072Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.702211755Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.702616759Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.702954954Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.703246639Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.703605559Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.704598275Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.705604368Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-26T04:55:22.053 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.706007181Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.715473363Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.715897434Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.716341846Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.719745366Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.720053981Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.720359726Z level=info msg="Executing migration" id="create folder table" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.720697704Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.72111129Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.721555662Z level=info msg="Executing migration" id="Update folder title length" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.721790913Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=migrator t=2024-04-26T04:55:21.722196457Z level=info msg="migrations completed" performed=485 skipped=0 duration=241.863518ms 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore t=2024-04-26T04:55:21.722579547Z level=info msg="Created default organization" 2024-04-26T04:55:22.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=plugin.loader t=2024-04-26T04:55:21.776638601Z level=info msg="Plugin registered" pluginID=input 2024-04-26T04:55:22.055 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=plugin.loader t=2024-04-26T04:55:21.805194211Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-26T04:55:22.055 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=plugin.loader t=2024-04-26T04:55:21.805225981Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-26T04:55:22.055 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=secrets t=2024-04-26T04:55:21.805260522Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-26T04:55:22.055 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=query_data t=2024-04-26T04:55:21.806939165Z level=info msg="Query Service initialization" 2024-04-26T04:55:22.055 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=live.push_http t=2024-04-26T04:55:21.809179199Z level=info msg="Live Push Gateway initialization" 2024-04-26T04:55:22.658 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-mon[31427]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:22.659 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.659 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.659 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.659 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.659 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-mon[31427]: Deploying daemon node-exporter.a on smithi152 2024-04-26T04:55:22.659 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=infra.usagestats.collector t=2024-04-26T04:55:22.525717896Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-26T04:55:22.659 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=provisioning.datasources t=2024-04-26T04:55:22.526125636Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-26T04:55:22.659 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=provisioning.datasources t=2024-04-26T04:55:22.558724022Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T04:55:22.659 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=provisioning.datasources t=2024-04-26T04:55:22.590467676Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=provisioning.alerting t=2024-04-26T04:55:22.612284032Z level=info msg="starting to provision alerting" 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=provisioning.alerting t=2024-04-26T04:55:22.612302819Z level=info msg="finished to provision alerting" 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=ngalert.state.manager t=2024-04-26T04:55:22.612383722Z level=info msg="Warming state cache for startup" 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=grafanaStorageLogger t=2024-04-26T04:55:22.612433986Z level=info msg="storage starting" 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=http.server t=2024-04-26T04:55:22.614163234Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore.transactions t=2024-04-26T04:55:22.623397219Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=ngalert.state.manager t=2024-04-26T04:55:22.623408649Z level=info msg="State cache has been initialized" states=0 duration=11.023797ms 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=ngalert.multiorg.alertmanager t=2024-04-26T04:55:22.623518771Z level=info msg="starting MultiOrg Alertmanager" 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=ticker t=2024-04-26T04:55:22.623548267Z level=info msg=starting first_tick=2024-04-26T04:55:30Z 2024-04-26T04:55:22.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore.transactions t=2024-04-26T04:55:22.658774914Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:55:22.879 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[39324]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[39324]: Deploying daemon node-exporter.a on smithi152 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[27560]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:22.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:22 smithi152 ceph-mon[27560]: Deploying daemon node-exporter.a on smithi152 2024-04-26T04:55:22.908 INFO:teuthology.orchestra.run.smithi152.stdout:34359738416 2024-04-26T04:55:22.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:55:23.030 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore.transactions t=2024-04-26T04:55:22.688602668Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:55:23.030 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore.transactions t=2024-04-26T04:55:22.699009469Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-26T04:55:23.030 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:55:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[62181]: logger=sqlstore.transactions t=2024-04-26T04:55:22.709299946Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-04-26T04:55:24.619 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:24 smithi152 ceph-mon[27560]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:24.619 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:24 smithi152 ceph-mon[39324]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:24.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:24 smithi195 ceph-mon[31427]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:25.044 INFO:teuthology.orchestra.run.smithi152.stdout:55834574891 2024-04-26T04:55:25.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:55:25.849 INFO:teuthology.orchestra.run.smithi152.stdout:111669149730 2024-04-26T04:55:25.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:55:26.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:26 smithi195 ceph-mon[31427]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:26.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:26 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:26.874 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:26 smithi152 ceph-mon[27560]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:26.874 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:26 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:26.875 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:26 smithi152 ceph-mon[39324]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:26.875 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:26 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:27.175 INFO:teuthology.orchestra.run.smithi152.stdout:201863462932 2024-04-26T04:55:27.175 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:55:28.491 INFO:teuthology.orchestra.run.smithi152.stdout:167503724569 2024-04-26T04:55:28.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:55:28.709 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:28 smithi195 ceph-mon[31427]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:28.802 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:28 smithi152 ceph-mon[27560]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:28.802 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:28 smithi152 ceph-mon[39324]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:29.109 INFO:teuthology.orchestra.run.smithi152.stdout:133143986206 2024-04-26T04:55:29.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:55:29.540 INFO:teuthology.orchestra.run.smithi152.stdout:77309411366 2024-04-26T04:55:29.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:55:29.545 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:30.408 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:30.658 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:30 smithi152 ceph-mon[27560]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:30.659 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:30 smithi152 ceph-mon[39324]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:30.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:30 smithi195 ceph-mon[31427]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:30.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:55:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:55:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:32.469 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:32.469 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:32.469 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:32.754 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:32 smithi152 ceph-mon[27560]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:32.754 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:32 smithi152 ceph-mon[39324]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:32.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:32 smithi195 ceph-mon[31427]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:33.630 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:33 smithi152 bash[73391]: Writing manifest to image destination 2024-04-26T04:55:33.881 INFO:teuthology.orchestra.run.smithi152.stdout:236223201295 2024-04-26T04:55:33.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:55:33.946 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:33 smithi152 podman[73391]: 2024-04-26 04:55:33.586668868 +0000 UTC m=+3.655559703 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:55:34.499 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:34 smithi152 podman[73391]: 2024-04-26 04:55:34.33707594 +0000 UTC m=+4.405966777 container create 84ac3154b2e4618cad9fbd1c9dcb71a13021fa31336dfa916bb987d9e5b6d4aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:34.777 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:34 smithi152 ceph-mon[27560]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:34.778 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:34 smithi152 ceph-mon[39324]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:34.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:34 smithi195 ceph-mon[31427]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:35.872 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 podman[73391]: 2024-04-26 04:55:35.587658659 +0000 UTC m=+5.656549494 container init 84ac3154b2e4618cad9fbd1c9dcb71a13021fa31336dfa916bb987d9e5b6d4aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:35.872 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 podman[73391]: 2024-04-26 04:55:35.594056028 +0000 UTC m=+5.662946863 container start 84ac3154b2e4618cad9fbd1c9dcb71a13021fa31336dfa916bb987d9e5b6d4aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:35.872 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.598Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.598Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.598Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.598Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:55:35.873 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.599Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:55:35.874 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:55:35.875 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:55:35.876 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:55:35.876 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:55:35.876 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:55:35.876 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:55:35.876 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:55:35.876 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:55:35.876 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[73753]: ts=2024-04-26T04:55:35.600Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 bash[73391]: 84ac3154b2e4618cad9fbd1c9dcb71a13021fa31336dfa916bb987d9e5b6d4aa 2024-04-26T04:55:35.877 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:55:35 smithi152 systemd[1]: Started Ceph node-exporter.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:36.338 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:36.718 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[27560]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[39324]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:36 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:36 smithi195 ceph-mon[31427]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:36 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:36 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:36 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:36 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:36.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:36 smithi195 bash[62458]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:55:37.505 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:37 smithi195 bash[62458]: Getting image source signatures 2024-04-26T04:55:37.505 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:37 smithi195 bash[62458]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:55:37.505 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:37 smithi195 bash[62458]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:55:37.505 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:37 smithi195 bash[62458]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:55:37.599 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:37 smithi152 ceph-mon[27560]: Deploying daemon node-exporter.b on smithi195 2024-04-26T04:55:37.600 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:37 smithi152 ceph-mon[39324]: Deploying daemon node-exporter.b on smithi195 2024-04-26T04:55:37.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:37 smithi195 ceph-mon[31427]: Deploying daemon node-exporter.b on smithi195 2024-04-26T04:55:38.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:38 smithi195 ceph-mon[31427]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:38.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:38 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/376750313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:55:38.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:38 smithi195 bash[62458]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:55:38.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:38 smithi195 bash[62458]: Writing manifest to image destination 2024-04-26T04:55:38.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:38 smithi152 ceph-mon[27560]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:38.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:38 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/376750313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:55:38.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:38 smithi152 ceph-mon[39324]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:38.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:38 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/376750313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:55:39.139 INFO:teuthology.orchestra.run.smithi152.stdout:201863462935 2024-04-26T04:55:39.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:38 smithi195 podman[62458]: 2024-04-26 04:55:38.774037524 +0000 UTC m=+2.149692052 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:55:39.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:38 smithi195 podman[62458]: 2024-04-26 04:55:38.864723875 +0000 UTC m=+2.240378447 container create b3a8846a9680afff27758d47e21d748edd643d318659f3a23f15c0f891f84c1e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:55:39.167 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 podman[62458]: 2024-04-26 04:55:39.075540528 +0000 UTC m=+2.451195055 container init b3a8846a9680afff27758d47e21d748edd643d318659f3a23f15c0f891f84c1e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 podman[62458]: 2024-04-26 04:55:39.081603803 +0000 UTC m=+2.457258331 container start b3a8846a9680afff27758d47e21d748edd643d318659f3a23f15c0f891f84c1e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.085Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.085Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.085Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.085Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:55:39.168 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:55:39.169 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:55:39.170 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:55:39.171 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.086Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.087Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[62509]: ts=2024-04-26T04:55:39.087Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:55:39.172 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 bash[62458]: b3a8846a9680afff27758d47e21d748edd643d318659f3a23f15c0f891f84c1e 2024-04-26T04:55:39.322 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:39.530 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:55:39 smithi195 systemd[1]: Started Ceph node-exporter.b for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[27560]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:40.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[39324]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:40.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:40 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-mon[31427]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:40.957 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:55:40 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:55:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:42 smithi152 ceph-mon[27560]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:42 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:42 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1692633895' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:55:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:42 smithi152 ceph-mon[39324]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:42 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:42 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1692633895' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:55:42.649 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:42 smithi195 ceph-mon[31427]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:42.649 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:42 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:42.649 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:42 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1692633895' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:55:43.382 INFO:teuthology.orchestra.run.smithi152.stdout:133143986208 2024-04-26T04:55:43.585 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462932 got 201863462935 for osd.6 2024-04-26T04:55:43.586 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:44.317 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:44.317 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:44.317 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[27560]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:44.317 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1175957815' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:55:44.318 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:44.318 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:44.318 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[39324]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:44.318 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:44 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1175957815' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:55:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:44 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:44 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:44 smithi195 ceph-mon[31427]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:44 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1175957815' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:55:44.744 INFO:teuthology.orchestra.run.smithi152.stdout:55834574895 2024-04-26T04:55:44.843 INFO:teuthology.orchestra.run.smithi152.stdout:167503724572 2024-04-26T04:55:45.281 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:45 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3843332210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:55:45.281 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:45 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3744487536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:55:45.281 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:45 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3843332210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:55:45.282 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:45 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3744487536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:55:45.306 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574895 for osd.1 2024-04-26T04:55:45.306 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:45 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3843332210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:55:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:45 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3744487536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:55:46.096 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724572 for osd.5 2024-04-26T04:55:46.096 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:46.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:46 smithi152 ceph-mon[27560]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:46 smithi152 ceph-mon[39324]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:46.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:46 smithi195 ceph-mon[31427]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:46.801 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986206 got 133143986208 for osd.4 2024-04-26T04:55:46.801 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:47.455 INFO:teuthology.orchestra.run.smithi152.stdout:111669149734 2024-04-26T04:55:47.557 INFO:teuthology.orchestra.run.smithi152.stdout:77309411371 2024-04-26T04:55:48.375 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149730 got 111669149734 for osd.3 2024-04-26T04:55:48.376 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:48.469 INFO:teuthology.orchestra.run.smithi152.stdout:236223201299 2024-04-26T04:55:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:48 smithi152 ceph-mon[39324]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:48 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/925408898' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:55:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:48 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1848607800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:55:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:48 smithi152 ceph-mon[27560]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:48 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/925408898' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:55:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:48 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1848607800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:55:48.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:48 smithi195 ceph-mon[31427]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:48.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:48 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/925408898' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:55:48.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:48 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1848607800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:55:49.016 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201299 for osd.7 2024-04-26T04:55:49.017 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:49.578 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411371 for osd.2 2024-04-26T04:55:49.578 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:49.736 INFO:teuthology.orchestra.run.smithi152.stdout:34359738422 2024-04-26T04:55:50.297 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738422 for osd.0 2024-04-26T04:55:50.298 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:50.298 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:55:50.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:55:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:50 smithi152 ceph-mon[27560]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:50 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4026835997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:55:50.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:50 smithi152 ceph-mon[39324]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:50.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:50 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/4026835997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:55:50.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:50 smithi195 ceph-mon[31427]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:50.684 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:50 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/4026835997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:55:51.017 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:51.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:55:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:55:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:51.448 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:51 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:51.449 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:51 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:51.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:51 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:52.464 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:52 smithi152 ceph-mon[27560]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:52.464 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:52 smithi152 ceph-mon[39324]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:52.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:52 smithi195 ceph-mon[31427]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:52.911 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:55:52.913 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-26T04:55:54.383 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-04-26T04:55:51.293420+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385572,"kb_used_data":6704,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575644,"statfs":{"total":767960285184,"available":765517459456,"internally_reserved":0,"allocated":6864896,"data_stored":3897115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001955"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.990186+0000","last_change":"2024-04-26T04:54:55.373997+0000","last_active":"2024-04-26T04:54:56.990186+0000","last_peered":"2024-04-26T04:54:56.990186+0000","last_clean":"2024-04-26T04:54:56.990186+0000","last_became_active":"2024-04-26T04:54:55.373681+0000","last_became_peered":"2024-04-26T04:54:55.373681+0000","last_unstale":"2024-04-26T04:54:56.990186+0000","last_undegraded":"2024-04-26T04:54:56.990186+0000","last_fullsized":"2024-04-26T04:54:56.990186+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:10:19.893941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.989501+0000","last_change":"2024-04-26T04:54:55.567081+0000","last_active":"2024-04-26T04:54:56.989501+0000","last_peered":"2024-04-26T04:54:56.989501+0000","last_clean":"2024-04-26T04:54:56.989501+0000","last_became_active":"2024-04-26T04:54:55.566744+0000","last_became_peered":"2024-04-26T04:54:55.566744+0000","last_unstale":"2024-04-26T04:54:56.989501+0000","last_undegraded":"2024-04-26T04:54:56.989501+0000","last_fullsized":"2024-04-26T04:54:56.989501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:46:08.438081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371522+0000","last_change":"2024-04-26T04:54:55.960283+0000","last_active":"2024-04-26T04:54:56.371522+0000","last_peered":"2024-04-26T04:54:56.371522+0000","last_clean":"2024-04-26T04:54:56.371522+0000","last_became_active":"2024-04-26T04:54:55.959970+0000","last_became_peered":"2024-04-26T04:54:55.959970+0000","last_unstale":"2024-04-26T04:54:56.371522+0000","last_undegraded":"2024-04-26T04:54:56.371522+0000","last_fullsized":"2024-04-26T04:54:56.371522+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:27:28.847430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371547+0000","last_change":"2024-04-26T04:54:55.960282+0000","last_active":"2024-04-26T04:54:56.371547+0000","last_peered":"2024-04-26T04:54:56.371547+0000","last_clean":"2024-04-26T04:54:56.371547+0000","last_became_active":"2024-04-26T04:54:55.959964+0000","last_became_peered":"2024-04-26T04:54:55.959964+0000","last_unstale":"2024-04-26T04:54:56.371547+0000","last_undegraded":"2024-04-26T04:54:56.371547+0000","last_fullsized":"2024-04-26T04:54:56.371547+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:56:37.017502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371507+0000","last_change":"2024-04-26T04:54:24.066215+0000","last_active":"2024-04-26T04:54:56.371507+0000","last_peered":"2024-04-26T04:54:56.371507+0000","last_clean":"2024-04-26T04:54:56.371507+0000","last_became_active":"2024-04-26T04:54:24.065870+0000","last_became_peered":"2024-04-26T04:54:24.065870+0000","last_unstale":"2024-04-26T04:54:56.371507+0000","last_undegraded":"2024-04-26T04:54:56.371507+0000","last_fullsized":"2024-04-26T04:54:56.371507+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_clean_scrub_stamp":"2024-04-26T04:52:21.376988+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:15:38.049930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.990062+0000","last_change":"2024-04-26T04:54:55.372814+0000","last_active":"2024-04-26T04:54:56.990062+0000","last_peered":"2024-04-26T04:54:56.990062+0000","last_clean":"2024-04-26T04:54:56.990062+0000","last_became_active":"2024-04-26T04:54:55.372511+0000","last_became_peered":"2024-04-26T04:54:55.372511+0000","last_unstale":"2024-04-26T04:54:56.990062+0000","last_undegraded":"2024-04-26T04:54:56.990062+0000","last_fullsized":"2024-04-26T04:54:56.990062+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:40.580236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371494+0000","last_change":"2024-04-26T04:54:55.960600+0000","last_active":"2024-04-26T04:54:56.371494+0000","last_peered":"2024-04-26T04:54:56.371494+0000","last_clean":"2024-04-26T04:54:56.371494+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:54:56.371494+0000","last_undegraded":"2024-04-26T04:54:56.371494+0000","last_fullsized":"2024-04-26T04:54:56.371494+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:08:33.905027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.990099+0000","last_change":"2024-04-26T04:54:55.372743+0000","last_active":"2024-04-26T04:54:56.990099+0000","last_peered":"2024-04-26T04:54:56.990099+0000","last_clean":"2024-04-26T04:54:56.990099+0000","last_became_active":"2024-04-26T04:54:55.372440+0000","last_became_peered":"2024-04-26T04:54:55.372440+0000","last_unstale":"2024-04-26T04:54:56.990099+0000","last_undegraded":"2024-04-26T04:54:56.990099+0000","last_fullsized":"2024-04-26T04:54:56.990099+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:02:35.773050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.989463+0000","last_change":"2024-04-26T04:54:55.960319+0000","last_active":"2024-04-26T04:54:56.989463+0000","last_peered":"2024-04-26T04:54:56.989463+0000","last_clean":"2024-04-26T04:54:56.989463+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:54:56.989463+0000","last_undegraded":"2024-04-26T04:54:56.989463+0000","last_fullsized":"2024-04-26T04:54:56.989463+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:23:09.651998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.39400000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.442}]},{"osd":2,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.55800000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.499},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.57499999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.58599999999999997}]},{"osd":5,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.62},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.52400000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.60599999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.60299999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.58299999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.505}]},{"osd":3,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.65600000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.61899999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.628}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.68999999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.34399999999999997,"15min":0.33000000000000002},"min":{"1min":0.25,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.53400000000000003,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.35399999999999998,"15min":0.32900000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.53500000000000003}]},{"osd":2,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.502,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.38900000000000001,"15min":0.38700000000000001},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.51900000000000002,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.51200000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.438,"15min":0.437},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.55200000000000005}]},{"osd":4,"last update":"Fri Apr 26 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56899999999999995,"15min":0.57599999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.74099999999999999,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59899999999999998,"15min":0.60899999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69799999999999995,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.65500000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:55:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.373,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69999999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.63},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58099999999999996,"15min":0.58399999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.78100000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.67900000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.755}]},{"osd":7,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.73599999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298564,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446588,"statfs":{"total":95995035648,"available":95689306112,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.33700000000000002},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.64000000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.55300000000000005,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.44400000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.42899999999999999,"15min":0.42599999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.59599999999999997,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.41799999999999998,"15min":0.40999999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.41499999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:55:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.46400000000000002,"15min":0.47599999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58999999999999997,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.42999999999999999,"15min":0.438},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.53700000000000003,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.39100000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.628,"15min":0.64100000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68300000000000005,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.625},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.629,"15min":0.63900000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.69299999999999995,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.66000000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:55:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59499999999999997,"15min":0.59599999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75900000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.55900000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.68600000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:55:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.747}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34599999999999997,"15min":0.33800000000000002},"min":{"1min":0.25700000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.45700000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.379},{"interface":"front","average":{"1min":0.375,"5min":0.36199999999999999,"15min":0.35999999999999999},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.51900000000000002,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.35599999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.35799999999999998},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53100000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.39800000000000002},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56699999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.42899999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.63500000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59699999999999998,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.42099999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.56200000000000006,"15min":0.55800000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.54900000000000004,"15min":0.54300000000000004},"min":{"1min":0.442,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.45800000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:55:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.58599999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:54:56 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.57299999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.64500000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.40300000000000002,"15min":0.40100000000000002},"min":{"1min":0.32100000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.501,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.442,"15min":0.439},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.34100000000000003}]},{"osd":1,"last update":"Fri Apr 26 04:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.41999999999999998,"15min":0.41299999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.32400000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.56699999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.32500000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.46300000000000002,"15min":0.46100000000000002},"min":{"1min":0.37,"5min":0.31,"15min":0.31},"max":{"1min":0.54500000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.57099999999999995,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.42299999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:55:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56799999999999995,"15min":0.57199999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.53700000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:55:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.59299999999999997,"15min":0.59999999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.58099999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.621}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.56699999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.32500000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.87,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.67200000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.42999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80700000000000005,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.623}]},{"osd":2,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.55300000000000005,"15min":0.55000000000000004},"min":{"1min":0.46300000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.60999999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.55600000000000005,"15min":0.55000000000000004},"min":{"1min":0.40100000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93200000000000005,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.66100000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.39800000000000002,"15min":0.39000000000000001},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.64800000000000002,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.34599999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.52000000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57199999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:55:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.69199999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.66400000000000003,"5min":0.748,"15min":0.748},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53100000000000003,"15min":0.52900000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.69699999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.622}]},{"osd":1,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53400000000000003,"15min":0.53300000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69399999999999995,"5min":0.753,"15min":0.753},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.50600000000000001,"15min":0.504},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.67700000000000005,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.65300000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.54700000000000004,"15min":0.54800000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.747,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.66300000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.39000000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.68899999999999995,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56399999999999995,"15min":0.56000000000000005},"min":{"1min":0.433,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72699999999999998,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.68500000000000005}]},{"osd":4,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39200000000000002,"15min":0.38700000000000001},"min":{"1min":0.315,"5min":0.253,"15min":0.253},"max":{"1min":0.60599999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.41199999999999998,"15min":0.40200000000000002},"min":{"1min":0.34200000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67700000000000005,"5min":0.75,"15min":0.75},"last":0.33000000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:54:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.46500000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.41199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:55:54.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:55:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:54 smithi152 ceph-mon[27560]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:55:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:54 smithi152 ceph-mon[27560]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:54 smithi152 ceph-mon[39324]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:55:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:54 smithi152 ceph-mon[39324]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:54 smithi195 ceph-mon[31427]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:55:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:54 smithi195 ceph-mon[31427]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:55.132 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:55:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[39324]: Reconfiguring daemon alertmanager.a on smithi152 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:55:56.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:56 smithi152 ceph-mon[27560]: Reconfiguring daemon alertmanager.a on smithi152 2024-04-26T04:55:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:56.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:56.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:55:56.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:55:56.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:56 smithi195 ceph-mon[31427]: Reconfiguring daemon alertmanager.a on smithi152 2024-04-26T04:55:57.945 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:55:57.947 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-26T04:55:58.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:58 smithi152 ceph-mon[27560]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:58 smithi152 ceph-mon[39324]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:58.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:58 smithi195 ceph-mon[31427]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:59.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:55:59 smithi152 ceph-mon[27560]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:55:59.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:55:59 smithi152 ceph-mon[39324]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:55:59.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:55:59 smithi195 ceph-mon[31427]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:55:59.956 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-04-26T04:55:57.294371+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385572,"kb_used_data":6704,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575644,"statfs":{"total":767960285184,"available":765517459456,"internally_reserved":0,"allocated":6864896,"data_stored":3897115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001778"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.990186+0000","last_change":"2024-04-26T04:54:55.373997+0000","last_active":"2024-04-26T04:54:56.990186+0000","last_peered":"2024-04-26T04:54:56.990186+0000","last_clean":"2024-04-26T04:54:56.990186+0000","last_became_active":"2024-04-26T04:54:55.373681+0000","last_became_peered":"2024-04-26T04:54:55.373681+0000","last_unstale":"2024-04-26T04:54:56.990186+0000","last_undegraded":"2024-04-26T04:54:56.990186+0000","last_fullsized":"2024-04-26T04:54:56.990186+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:10:19.893941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.989501+0000","last_change":"2024-04-26T04:54:55.567081+0000","last_active":"2024-04-26T04:54:56.989501+0000","last_peered":"2024-04-26T04:54:56.989501+0000","last_clean":"2024-04-26T04:54:56.989501+0000","last_became_active":"2024-04-26T04:54:55.566744+0000","last_became_peered":"2024-04-26T04:54:55.566744+0000","last_unstale":"2024-04-26T04:54:56.989501+0000","last_undegraded":"2024-04-26T04:54:56.989501+0000","last_fullsized":"2024-04-26T04:54:56.989501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:46:08.438081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371522+0000","last_change":"2024-04-26T04:54:55.960283+0000","last_active":"2024-04-26T04:54:56.371522+0000","last_peered":"2024-04-26T04:54:56.371522+0000","last_clean":"2024-04-26T04:54:56.371522+0000","last_became_active":"2024-04-26T04:54:55.959970+0000","last_became_peered":"2024-04-26T04:54:55.959970+0000","last_unstale":"2024-04-26T04:54:56.371522+0000","last_undegraded":"2024-04-26T04:54:56.371522+0000","last_fullsized":"2024-04-26T04:54:56.371522+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:27:28.847430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371547+0000","last_change":"2024-04-26T04:54:55.960282+0000","last_active":"2024-04-26T04:54:56.371547+0000","last_peered":"2024-04-26T04:54:56.371547+0000","last_clean":"2024-04-26T04:54:56.371547+0000","last_became_active":"2024-04-26T04:54:55.959964+0000","last_became_peered":"2024-04-26T04:54:55.959964+0000","last_unstale":"2024-04-26T04:54:56.371547+0000","last_undegraded":"2024-04-26T04:54:56.371547+0000","last_fullsized":"2024-04-26T04:54:56.371547+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:56:37.017502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371507+0000","last_change":"2024-04-26T04:54:24.066215+0000","last_active":"2024-04-26T04:54:56.371507+0000","last_peered":"2024-04-26T04:54:56.371507+0000","last_clean":"2024-04-26T04:54:56.371507+0000","last_became_active":"2024-04-26T04:54:24.065870+0000","last_became_peered":"2024-04-26T04:54:24.065870+0000","last_unstale":"2024-04-26T04:54:56.371507+0000","last_undegraded":"2024-04-26T04:54:56.371507+0000","last_fullsized":"2024-04-26T04:54:56.371507+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_clean_scrub_stamp":"2024-04-26T04:52:21.376988+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:15:38.049930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.990062+0000","last_change":"2024-04-26T04:54:55.372814+0000","last_active":"2024-04-26T04:54:56.990062+0000","last_peered":"2024-04-26T04:54:56.990062+0000","last_clean":"2024-04-26T04:54:56.990062+0000","last_became_active":"2024-04-26T04:54:55.372511+0000","last_became_peered":"2024-04-26T04:54:55.372511+0000","last_unstale":"2024-04-26T04:54:56.990062+0000","last_undegraded":"2024-04-26T04:54:56.990062+0000","last_fullsized":"2024-04-26T04:54:56.990062+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:40.580236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.371494+0000","last_change":"2024-04-26T04:54:55.960600+0000","last_active":"2024-04-26T04:54:56.371494+0000","last_peered":"2024-04-26T04:54:56.371494+0000","last_clean":"2024-04-26T04:54:56.371494+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:54:56.371494+0000","last_undegraded":"2024-04-26T04:54:56.371494+0000","last_fullsized":"2024-04-26T04:54:56.371494+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:08:33.905027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.990099+0000","last_change":"2024-04-26T04:54:55.372743+0000","last_active":"2024-04-26T04:54:56.990099+0000","last_peered":"2024-04-26T04:54:56.990099+0000","last_clean":"2024-04-26T04:54:56.990099+0000","last_became_active":"2024-04-26T04:54:55.372440+0000","last_became_peered":"2024-04-26T04:54:55.372440+0000","last_unstale":"2024-04-26T04:54:56.990099+0000","last_undegraded":"2024-04-26T04:54:56.990099+0000","last_fullsized":"2024-04-26T04:54:56.990099+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:02:35.773050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:56.989463+0000","last_change":"2024-04-26T04:54:55.960319+0000","last_active":"2024-04-26T04:54:56.989463+0000","last_peered":"2024-04-26T04:54:56.989463+0000","last_clean":"2024-04-26T04:54:56.989463+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:54:56.989463+0000","last_undegraded":"2024-04-26T04:54:56.989463+0000","last_fullsized":"2024-04-26T04:54:56.989463+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:23:09.651998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.52300000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.69799999999999995}]},{"osd":2,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.58599999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.66900000000000004}]},{"osd":4,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.68600000000000005}]},{"osd":5,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.73699999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.76000000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.55100000000000005}]},{"osd":1,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.63900000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.66200000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.502},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.621}]},{"osd":4,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.754},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.69499999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.79000000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.77100000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.34399999999999997,"15min":0.33000000000000002},"min":{"1min":0.25,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.53400000000000003,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.35399999999999998,"15min":0.32900000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.249}]},{"osd":2,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.502,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.38900000000000001,"15min":0.38700000000000001},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.51900000000000002,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.27800000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:55:48 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.437,"15min":0.441},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.56699999999999995,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.314},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.42799999999999999,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55200000000000005,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.41699999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56899999999999995,"15min":0.57599999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.74099999999999999,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59899999999999998,"15min":0.60899999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69799999999999995,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.66000000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:55:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.373,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69999999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58099999999999996,"15min":0.58399999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.78100000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.57699999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.65000000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.59799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298564,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446588,"statfs":{"total":95995035648,"available":95689306112,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.33700000000000002},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.64000000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.55300000000000005,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.36199999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.42899999999999999,"15min":0.42599999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.59599999999999997,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.41799999999999998,"15min":0.40999999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.30099999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:55:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.46400000000000002,"15min":0.47599999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58999999999999997,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.378},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.42999999999999999,"15min":0.438},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.53700000000000003,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.39000000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.628,"15min":0.64100000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68300000000000005,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.752},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.629,"15min":0.63900000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.69299999999999995,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.66600000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:55:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59499999999999997,"15min":0.59599999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75900000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.72399999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.68999999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:55:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.76300000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34599999999999997,"15min":0.33800000000000002},"min":{"1min":0.25700000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.45700000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.375,"5min":0.36199999999999999,"15min":0.35999999999999999},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.51900000000000002,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.29499999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.35799999999999998},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53100000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.312},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.39800000000000002},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56699999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.35399999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:55:54 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.46500000000000002,"15min":0.46899999999999997},"min":{"1min":0.34599999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.60999999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.45000000000000001,"15min":0.45600000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.58599999999999997,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.34000000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.56200000000000006,"15min":0.55800000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.54900000000000004,"15min":0.54300000000000004},"min":{"1min":0.442,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.47199999999999998}]},{"osd":5,"last update":"Fri Apr 26 04:55:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.83099999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:54:56 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.629}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.77500000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:53 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.39100000000000001,"15min":0.39700000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.52700000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.42699999999999999,"15min":0.434},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54600000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.36099999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:55:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.41199999999999998,"15min":0.40999999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.41899999999999998,"15min":0.42599999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.27500000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:55:53 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.31},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.441,"15min":0.45600000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.48199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.42099999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:55:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56799999999999995,"15min":0.57199999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.44500000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:55:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.59299999999999997,"15min":0.59999999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.56100000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.47499999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.56999999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.32500000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.87,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.64200000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.42999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80700000000000005,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.71699999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.55300000000000005,"15min":0.55000000000000004},"min":{"1min":0.46300000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.67600000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.55600000000000005,"15min":0.55000000000000004},"min":{"1min":0.40100000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93200000000000005,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.65900000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.39800000000000002,"15min":0.39000000000000001},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.64800000000000002,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.34599999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.439}]},{"osd":6,"last update":"Fri Apr 26 04:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58499999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:55:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.46500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.66400000000000003,"5min":0.748,"15min":0.748},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53100000000000003,"15min":0.52900000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.69699999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.66600000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53400000000000003,"15min":0.53300000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69399999999999995,"5min":0.753,"15min":0.753},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.50600000000000001,"15min":0.504},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.67700000000000005,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45000000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.54700000000000004,"15min":0.54800000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.747,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.48299999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.39000000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.68899999999999995,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.502},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56399999999999995,"15min":0.56000000000000005},"min":{"1min":0.433,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72699999999999998,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.63400000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39200000000000002,"15min":0.38700000000000001},"min":{"1min":0.315,"5min":0.253,"15min":0.253},"max":{"1min":0.60599999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.378},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.41199999999999998,"15min":0.40200000000000002},"min":{"1min":0.34200000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67700000000000005,"5min":0.75,"15min":0.75},"last":0.57999999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:54:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.53000000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:55:59.957 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:55:59.957 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:55:59.958 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:55:59.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:00.557 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:00.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:00 smithi152 ceph-mon[27560]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:00.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:00 smithi152 systemd[1]: Stopping Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:56:00.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:00 smithi152 ceph-mon[39324]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:00.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:00 smithi195 ceph-mon[31427]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:01.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:56:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:01.612 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:01 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[70920]: ts=2024-04-26T04:56:01.360Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:56:01.612 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:01 smithi152 podman[76185]: 2024-04-26 04:56:01.381525033 +0000 UTC m=+0.538775093 container died 1e69262794a8e083b883d8759d5fb737d5bdfd4f0c8a635ba61b052d1170cf6f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:01.879 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:01 smithi152 podman[76185]: 2024-04-26 04:56:01.702900266 +0000 UTC m=+0.860150331 container cleanup 1e69262794a8e083b883d8759d5fb737d5bdfd4f0c8a635ba61b052d1170cf6f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:01.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:01 smithi152 bash[76185]: ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:56:02.224 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:01 smithi152 podman[76222]: 2024-04-26 04:56:01.938336323 +0000 UTC m=+0.555899608 container remove 1e69262794a8e083b883d8759d5fb737d5bdfd4f0c8a635ba61b052d1170cf6f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:02.225 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:02 smithi152 podman[76222]: 2024-04-26 04:56:02.013687677 +0000 UTC m=+0.631250917 volume remove c049d311c606be0a31e1dd4d4a563fc42ace760695e36ce17d428bae807812cb 2024-04-26T04:56:02.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:02 smithi152 ceph-mon[27560]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:02.517 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:02 smithi152 ceph-mon[39324]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:02.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:02 smithi195 ceph-mon[31427]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:03.129 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:02 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:56:03.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:02 smithi152 systemd[1]: Stopped Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:56:03.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:02 smithi152 systemd[1]: Starting Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:56:03.308 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:56:03.308 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:56:03.612 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4290068266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:03.613 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/4290068266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:03.613 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 podman[76398]: 2024-04-26 04:56:03.151641165 +0000 UTC m=+0.018417481 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:56:03.613 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 podman[76398]: 2024-04-26 04:56:03.271311618 +0000 UTC m=+0.138087927 volume create 71b2c167b01e88e58a0c1012018b95eab3d4011a2d4440397b32e581a2c4f46c 2024-04-26T04:56:03.613 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 podman[76398]: 2024-04-26 04:56:03.405387362 +0000 UTC m=+0.272163716 container create 92be36185dbe12bd7a282534332ca68385755ec0e04bc35705fff08b6e6a3c07 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:03.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:03 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/4290068266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:04.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 podman[76398]: 2024-04-26 04:56:03.73322168 +0000 UTC m=+0.599997989 container init 92be36185dbe12bd7a282534332ca68385755ec0e04bc35705fff08b6e6a3c07 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:04.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 podman[76398]: 2024-04-26 04:56:03.738742619 +0000 UTC m=+0.605518928 container start 92be36185dbe12bd7a282534332ca68385755ec0e04bc35705fff08b6e6a3c07 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:04.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.750Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:56:04.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.750Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:56:04.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.752Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.152 port=9094 2024-04-26T04:56:04.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.752Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:56:04.130 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.775Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:56:04.131 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.776Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:56:04.131 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.778Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:56:04.131 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:03.778Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:56:04.131 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 bash[76398]: 92be36185dbe12bd7a282534332ca68385755ec0e04bc35705fff08b6e6a3c07 2024-04-26T04:56:04.131 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:03 smithi152 systemd[1]: Started Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:56:04.470 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:04 smithi152 ceph-mon[27560]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:04.471 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:04 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:04.471 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:04 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:04.471 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:04 smithi152 ceph-mon[39324]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:04.471 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:04 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:04.471 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:04 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:04.497 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:56:04.497 INFO:tasks.cephadm:Setup complete, yielding 2024-04-26T04:56:04.498 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T04:56:04.507 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-26T04:56:04.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-26T04:56:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:04 smithi195 ceph-mon[31427]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:04 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:04 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:04.762 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:05.518 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:05 smithi195 ceph-mon[31427]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:56:05.518 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:05 smithi195 ceph-mon[31427]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:56:05.629 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:05 smithi152 ceph-mon[39324]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:56:05.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:05 smithi152 ceph-mon[39324]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:56:05.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:05 smithi152 ceph-mon[27560]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:56:05.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:05 smithi152 ceph-mon[27560]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:56:06.029 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:05 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:05.753Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000082153s 2024-04-26T04:56:06.500 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:06 smithi195 ceph-mon[31427]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:06.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:06 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:06.629 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:06 smithi152 ceph-mon[39324]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:06.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:06 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:06 smithi152 ceph-mon[27560]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:06 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:07.199 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:06 smithi195 systemd[1]: Stopping Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:56:07.522 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:56:07.523 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:56:07.523 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.198Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:56:07.523 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:56:07.523 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:56:07.523 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:56:07.523 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:56:07.523 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:56:07.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:56:07.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:56:07.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:56:07.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:56:07.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:56:07.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[59926]: ts=2024-04-26T04:56:07.199Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:56:07.524 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 podman[63397]: 2024-04-26 04:56:07.209474191 +0000 UTC m=+0.121552920 container died 1ed61ab11bc29a675a4dce71517782cbcfeef1a5eb5da56e00b0b684c5540348 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:07.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:07 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:56:07.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:07 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:56:07.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:07 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:56:07.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:07 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:56:07.780 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 podman[63397]: 2024-04-26 04:56:07.62996608 +0000 UTC m=+0.542044797 container cleanup 1ed61ab11bc29a675a4dce71517782cbcfeef1a5eb5da56e00b0b684c5540348 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:07.780 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 bash[63397]: ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:56:08.270 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:07 smithi195 podman[63417]: 2024-04-26 04:56:07.814331818 +0000 UTC m=+0.608256229 container remove 1ed61ab11bc29a675a4dce71517782cbcfeef1a5eb5da56e00b0b684c5540348 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:08.271 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:56:08.271 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 systemd[1]: Stopped Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:56:08.271 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.133s CPU time. 2024-04-26T04:56:08.271 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 systemd[1]: Starting Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:56:08.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-mon[31427]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:08.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-mon[31427]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:08.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:56:08.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-mon[31427]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:56:08.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:08.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:08.531 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 podman[63518]: 2024-04-26 04:56:08.175052816 +0000 UTC m=+0.018959713 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:56:08.531 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 podman[63518]: 2024-04-26 04:56:08.298603781 +0000 UTC m=+0.142510656 container create a731aeae0e891d1d14070bfd4344f27ab80c8356f42f5ed9ab81fb7fc4ab001b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:08.754 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[27560]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[27560]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[27560]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a[27537]: 2024-04-26T04:56:08.400+0000 7f7910407700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[39324]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[39324]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[39324]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:08.755 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 podman[63518]: 2024-04-26 04:56:08.534216773 +0000 UTC m=+0.378123665 container init a731aeae0e891d1d14070bfd4344f27ab80c8356f42f5ed9ab81fb7fc4ab001b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 podman[63518]: 2024-04-26 04:56:08.539505022 +0000 UTC m=+0.383411901 container start a731aeae0e891d1d14070bfd4344f27ab80c8356f42f5ed9ab81fb7fc4ab001b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:56:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.563Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:56:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.563Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:56:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.563Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi195 (none))" 2024-04-26T04:56:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.563Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.563Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.565Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.565Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.567Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.567Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.568Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.568Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.925µs 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.568Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.569Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.569Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-26T04:56:09.032 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.569Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.57µs wal_replay_duration=529.246µs wbl_replay_duration=100ns total_replay_duration=565.947µs 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.570Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.570Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.570Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.585Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.391706ms db_storage=840ns remote_storage=1.387µs web_handler=340ns query_engine=632ns scrape=468.736µs scrape_sd=22.194µs notify=30.514µs notify_sd=6.257µs rules=13.648085ms tracing=5.643µs 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.585Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:56:08.585Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 bash[63518]: a731aeae0e891d1d14070bfd4344f27ab80c8356f42f5ed9ab81fb7fc4ab001b 2024-04-26T04:56:09.033 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:56:08 smithi195 systemd[1]: Started Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:56:09.101 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STOPPING 2024-04-26T04:56:09.101 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:56:09.101 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STOPPED 2024-04-26T04:56:09.101 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STARTING 2024-04-26T04:56:09.101 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Serving on http://:::9283 2024-04-26T04:56:09.101 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STARTED 2024-04-26T04:56:09.101 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STOPPING 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STOPPED 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STARTING 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Serving on http://:::9283 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STARTED 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STOPPING 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STOPPED 2024-04-26T04:56:09.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:08] ENGINE Bus STARTING 2024-04-26T04:56:09.103 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:09] ENGINE Serving on http://:::9283 2024-04-26T04:56:09.103 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:56:09] ENGINE Bus STARTED 2024-04-26T04:56:09.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-26T04:56:09.646 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: fsmap foo:0 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: Saving service mds.foo spec with placement count:2 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.647 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152:9093"}]: dispatch 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.648 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:09 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:09.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:56:09.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:56:09.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:09.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:09.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:56:09.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:56:09.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: fsmap foo:0 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: Saving service mds.foo spec with placement count:2 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152:9093"}]: dispatch 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:56:09.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:56:09.651 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: fsmap foo:0 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: Saving service mds.foo spec with placement count:2 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152:9093"}]: dispatch 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:56:09.652 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:56:09.653 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:09.653 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:09 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:10.073 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:10.504 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:56:10.504 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152:9093"}]: dispatch 2024-04-26T04:56:10.504 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:56:10.504 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[27560]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152:9093"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:56:10.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:56:10.506 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:10.506 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:10 smithi152 ceph-mon[39324]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:56:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:56:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152:9093"}]: dispatch 2024-04-26T04:56:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:56:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:56:10.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:56:10.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:56:10.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:10.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:10 smithi195 ceph-mon[31427]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:56:11.662 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:11 smithi152 ceph-mon[27560]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:56:11.662 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:11 smithi152 ceph-mon[39324]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:56:11.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:11 smithi195 ceph-mon[31427]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:56:12.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:12 smithi152 ceph-mon[27560]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:12.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:12 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:12.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:12 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:12.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:12 smithi152 ceph-mon[39324]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:12.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:12 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:12.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:12 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:12 smithi195 ceph-mon[31427]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:12 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:12 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:12.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-26T04:56:13.879 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:56:13 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:56:13.756Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003868719s 2024-04-26T04:56:14.049 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:56:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:14 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:14.879 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:56:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:14 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:15.728 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:15 smithi152 ceph-mon[27560]: Deploying daemon mds.foo.smithi152.rnhpxy on smithi152 2024-04-26T04:56:15.728 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:15 smithi152 ceph-mon[39324]: Deploying daemon mds.foo.smithi152.rnhpxy on smithi152 2024-04-26T04:56:15.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:15 smithi195 ceph-mon[31427]: Deploying daemon mds.foo.smithi152.rnhpxy on smithi152 2024-04-26T04:56:16.698 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:16 smithi152 ceph-mon[27560]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:16.699 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:16 smithi152 ceph-mon[39324]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:16.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:16 smithi195 ceph-mon[31427]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:18.450 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T04:56:18.459 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:56:18.459 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:56:18.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:56:18.652 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:18.652 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.652 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:18.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:18.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:56:18.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:18 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:18.717 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:56:18.840 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:56:18.841 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:18 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: Deploying daemon mds.foo.smithi195.bqeurw on smithi195 2024-04-26T04:56:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:boot 2024-04-26T04:56:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: daemon mds.foo.smithi152.rnhpxy assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:56:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:56:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:56:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: Cluster is now healthy 2024-04-26T04:56:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: fsmap foo:0 1 up:standby 2024-04-26T04:56:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:56:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:creating} 2024-04-26T04:56:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:19 smithi195 ceph-mon[31427]: daemon mds.foo.smithi152.rnhpxy is now active in filesystem foo as rank 0 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: Deploying daemon mds.foo.smithi195.bqeurw on smithi195 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:boot 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: daemon mds.foo.smithi152.rnhpxy assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: Cluster is now healthy 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: fsmap foo:0 1 up:standby 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:creating} 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[27560]: daemon mds.foo.smithi152.rnhpxy is now active in filesystem foo as rank 0 2024-04-26T04:56:19.821 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: Deploying daemon mds.foo.smithi195.bqeurw on smithi195 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:boot 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: daemon mds.foo.smithi152.rnhpxy assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: Cluster is now healthy 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: fsmap foo:0 1 up:standby 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:creating} 2024-04-26T04:56:19.822 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:19 smithi152 ceph-mon[39324]: daemon mds.foo.smithi152.rnhpxy is now active in filesystem foo as rank 0 2024-04-26T04:56:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:20 smithi195 ceph-mon[31427]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:20 smithi195 ceph-mon[31427]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:active 2024-04-26T04:56:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:20 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 2024-04-26T04:56:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:20 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:20 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:20 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:56:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[27560]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[27560]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:active 2024-04-26T04:56:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[27560]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 2024-04-26T04:56:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[39324]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:20.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[39324]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:active 2024-04-26T04:56:20.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[39324]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 2024-04-26T04:56:20.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:20.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:20 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:56:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:21.468 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":20,"flags":0,"active_gid":14430,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1036587324},{"type":"v1","addr":"172.21.15.152:6801","nonce":1036587324}]},"active_addr":"172.21.15.152:6801/1036587324","active_change":"2024-04-26T04:54:51.271327+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.152:8443/","prometheus":"http://172.21.15.152:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3664778945}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3677441985}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":1107620726}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3193673978}]}]} 2024-04-26T04:56:21.473 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:56:21.474 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:56:21.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:56:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:56:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: mds.? [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] up:boot 2024-04-26T04:56:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1442890125' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:21 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.799 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.799 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:56:21.799 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: mds.? [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] up:boot 2024-04-26T04:56:21.799 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:21.799 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:21.799 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.799 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1442890125' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: mds.? [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] up:boot 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.800 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1442890125' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.801 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:21.801 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:21 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:22.241 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:22.365 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:22 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:22] "GET /metrics HTTP/1.1" 200 238796 "" "Prometheus/2.43.0" 2024-04-26T04:56:22.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:22 smithi152 ceph-mon[27560]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:22.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:22 smithi152 ceph-mon[39324]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:22 smithi195 ceph-mon[31427]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:23.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:23 smithi195 ceph-mon[31427]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:active 2024-04-26T04:56:23.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:23 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:23.818 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:23 smithi152 ceph-mon[27560]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:active 2024-04-26T04:56:23.818 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:23 smithi152 ceph-mon[27560]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:23.818 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:23 smithi152 ceph-mon[39324]: mds.? [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] up:active 2024-04-26T04:56:23.818 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:23 smithi152 ceph-mon[39324]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:24.254 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:56:24.255 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":69,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T04:56:10.409692+0000","last_up_change":"2024-04-26T04:54:20.182998+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:54:53.698027+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:56:07.054314+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:56:07.629218+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6803","nonce":354037778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6805","nonce":354037778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6809","nonce":354037778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6807","nonce":354037778}]},"public_addr":"172.21.15.152:6803/354037778","cluster_addr":"172.21.15.152:6805/354037778","heartbeat_back_addr":"172.21.15.152:6809/354037778","heartbeat_front_addr":"172.21.15.152:6807/354037778","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6811","nonce":1928910910}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6813","nonce":1928910910}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6817","nonce":1928910910}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6815","nonce":1928910910}]},"public_addr":"172.21.15.152:6811/1928910910","cluster_addr":"172.21.15.152:6813/1928910910","heartbeat_back_addr":"172.21.15.152:6817/1928910910","heartbeat_front_addr":"172.21.15.152:6815/1928910910","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6819","nonce":2274658721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6821","nonce":2274658721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6825","nonce":2274658721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6823","nonce":2274658721}]},"public_addr":"172.21.15.152:6819/2274658721","cluster_addr":"172.21.15.152:6821/2274658721","heartbeat_back_addr":"172.21.15.152:6825/2274658721","heartbeat_front_addr":"172.21.15.152:6823/2274658721","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6827","nonce":816595688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6829","nonce":816595688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6833","nonce":816595688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6831","nonce":816595688}]},"public_addr":"172.21.15.152:6827/816595688","cluster_addr":"172.21.15.152:6829/816595688","heartbeat_back_addr":"172.21.15.152:6833/816595688","heartbeat_front_addr":"172.21.15.152:6831/816595688","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:6800/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/604484318":"2024-04-27T04:54:51.271123+0000","172.21.15.152:6801/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/2636634813":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1523725454":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2161488710":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[27560]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/353536638' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[39324]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/353536638' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:24.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:24 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:24 smithi195 ceph-mon[31427]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:24 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/353536638' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:56:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:24 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:24 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:25.095 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:56:25.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:56:25.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:25 smithi152 ceph-mon[27560]: mds.? [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] up:standby 2024-04-26T04:56:25.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:25 smithi152 ceph-mon[27560]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:25.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:25 smithi152 ceph-mon[39324]: mds.? [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] up:standby 2024-04-26T04:56:25.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:25 smithi152 ceph-mon[39324]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:25.688 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:25.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:25 smithi195 ceph-mon[31427]: mds.? [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] up:standby 2024-04-26T04:56:25.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:25 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T04:56:26.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:26 smithi152 ceph-mon[27560]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:26.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:26 smithi152 ceph-mon[39324]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:26.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:26 smithi195 ceph-mon[31427]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:28.249 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:56:28.250 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":69,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T04:56:10.409692+0000","last_up_change":"2024-04-26T04:54:20.182998+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:54:53.698027+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:56:07.054314+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:56:07.629218+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6803","nonce":354037778}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6805","nonce":354037778}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6809","nonce":354037778}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":354037778},{"type":"v1","addr":"172.21.15.152:6807","nonce":354037778}]},"public_addr":"172.21.15.152:6803/354037778","cluster_addr":"172.21.15.152:6805/354037778","heartbeat_back_addr":"172.21.15.152:6809/354037778","heartbeat_front_addr":"172.21.15.152:6807/354037778","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6811","nonce":1928910910}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6813","nonce":1928910910}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6817","nonce":1928910910}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1928910910},{"type":"v1","addr":"172.21.15.152:6815","nonce":1928910910}]},"public_addr":"172.21.15.152:6811/1928910910","cluster_addr":"172.21.15.152:6813/1928910910","heartbeat_back_addr":"172.21.15.152:6817/1928910910","heartbeat_front_addr":"172.21.15.152:6815/1928910910","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6819","nonce":2274658721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6821","nonce":2274658721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6825","nonce":2274658721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2274658721},{"type":"v1","addr":"172.21.15.152:6823","nonce":2274658721}]},"public_addr":"172.21.15.152:6819/2274658721","cluster_addr":"172.21.15.152:6821/2274658721","heartbeat_back_addr":"172.21.15.152:6825/2274658721","heartbeat_front_addr":"172.21.15.152:6823/2274658721","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6827","nonce":816595688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6829","nonce":816595688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6833","nonce":816595688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":816595688},{"type":"v1","addr":"172.21.15.152:6831","nonce":816595688}]},"public_addr":"172.21.15.152:6827/816595688","cluster_addr":"172.21.15.152:6829/816595688","heartbeat_back_addr":"172.21.15.152:6833/816595688","heartbeat_front_addr":"172.21.15.152:6831/816595688","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:6800/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/604484318":"2024-04-27T04:54:51.271123+0000","172.21.15.152:6801/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/2636634813":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1523725454":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/2161488710":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:56:28.341 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.341 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.341 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:28.341 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[27560]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:28.342 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:28 smithi152 ceph-mon[39324]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:28 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:28 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:28 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:28 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:28 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:28 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:28.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:28 smithi195 ceph-mon[31427]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:28.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:56:28.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:56:28.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:56:28.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:56:28.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:56:28.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:56:28.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:56:28.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:56:29.306 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:29 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3324262159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:56:29.306 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:29 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3324262159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:56:29.505 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:29 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3324262159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:56:29.679 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:29.685 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:29.685 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:29.686 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:29.689 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:29.690 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:29.741 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:30.354 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:30 smithi152 ceph-mon[27560]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:30.354 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:30 smithi152 ceph-mon[39324]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:30.368 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:30 smithi195 ceph-mon[31427]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:31.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:56:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:32.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:31 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:32.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:31 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:32.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:31 smithi195 ceph-mon[31427]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:32.125 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:31 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:32.125 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:31 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:32.125 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:31 smithi152 ceph-mon[27560]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:32.125 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:31 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:32.125 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:31 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:32.125 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:31 smithi152 ceph-mon[39324]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:32.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:32] "GET /metrics HTTP/1.1" 200 238796 "" "Prometheus/2.43.0" 2024-04-26T04:56:34.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:34 smithi152 ceph-mon[27560]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:34.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:34 smithi152 ceph-mon[39324]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:34.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:34 smithi195 ceph-mon[31427]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:56:35.311 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:36.459 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:36 smithi152 ceph-mon[27560]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:56:36.459 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:36 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:36.459 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:36 smithi152 ceph-mon[39324]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:56:36.459 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:36 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:36 smithi195 ceph-mon[31427]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:56:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:36 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:38.626 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:38 smithi152 ceph-mon[27560]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:56:38.626 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:38 smithi152 ceph-mon[39324]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:56:38.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:38 smithi195 ceph-mon[31427]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:56:39.547 INFO:teuthology.orchestra.run.smithi152.stdout:55834574908 2024-04-26T04:56:39.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:56:40.312 INFO:teuthology.orchestra.run.smithi152.stdout:201863462948 2024-04-26T04:56:40.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:56:40.629 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:40 smithi152 ceph-mon[39324]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:40.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:40 smithi152 ceph-mon[27560]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:40.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:40 smithi195 ceph-mon[31427]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:40.719 INFO:teuthology.orchestra.run.smithi152.stdout:111669149745 2024-04-26T04:56:40.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:56:41.025 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:56:40 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:42.287 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:42.364 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:42 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:42] "GET /metrics HTTP/1.1" 200 239094 "" "Prometheus/2.43.0" 2024-04-26T04:56:42.622 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:42 smithi152 ceph-mon[39324]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:42.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:42 smithi152 ceph-mon[27560]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:42.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:42 smithi195 ceph-mon[31427]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:44.629 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:44 smithi152 ceph-mon[39324]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:44 smithi152 ceph-mon[27560]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:44.776 INFO:teuthology.orchestra.run.smithi152.stdout:77309411383 2024-04-26T04:56:44.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:56:44.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:44 smithi195 ceph-mon[31427]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:46.079 INFO:teuthology.orchestra.run.smithi152.stdout:133143986222 2024-04-26T04:56:46.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:56:46.749 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[27560]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[39324]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:46.750 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:46 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:47.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:46 smithi195 ceph-mon[31427]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:47.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:46 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:47.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:46 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:47.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:46 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:47.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:46 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:47.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:46 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:56:48.178 INFO:teuthology.orchestra.run.smithi152.stdout:236223201311 2024-04-26T04:56:48.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:56:48.429 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:48.429 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:48.832 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:48 smithi152 ceph-mon[27560]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:48.832 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:48 smithi152 ceph-mon[39324]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:48.854 INFO:teuthology.orchestra.run.smithi152.stdout:34359738434 2024-04-26T04:56:48.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:56:49.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:48 smithi195 ceph-mon[31427]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:49.495 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:49.496 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:50.657 INFO:teuthology.orchestra.run.smithi152.stdout:167503724586 2024-04-26T04:56:50.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:56:50.863 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:50 smithi152 ceph-mon[27560]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:50.863 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:50 smithi152 ceph-mon[39324]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:51.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:50 smithi195 ceph-mon[31427]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:51.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:56:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:51.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-26T04:56:51.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:51 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:52.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:51 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:56:52.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:51 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T04:56:52.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:51 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:56:52.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:51 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-26T04:56:52.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:51 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T04:56:52.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:51 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-26T04:56:52.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:51 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:52.526 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:56:52] "GET /metrics HTTP/1.1" 200 239059 "" "Prometheus/2.43.0" 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[27560]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[27560]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[39324]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-26T04:56:52.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-26T04:56:52.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:52 smithi152 ceph-mon[39324]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:56:53.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:52 smithi195 ceph-mon[31427]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:53.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:52 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-26T04:56:53.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:52 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-26T04:56:53.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:52 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-26T04:56:53.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:52 smithi195 ceph-mon[31427]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:56:53.834 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:53 smithi152 ceph-mon[27560]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:56:53.834 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:53 smithi152 ceph-mon[39324]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:56:54.008 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:54.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:53 smithi195 ceph-mon[31427]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:56:54.785 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:54 smithi152 ceph-mon[39324]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:54.785 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:54 smithi152 ceph-mon[27560]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:55.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:54 smithi195 ceph-mon[31427]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:55.149 INFO:teuthology.orchestra.run.smithi152.stdout:111669149749 2024-04-26T04:56:55.335 INFO:teuthology.orchestra.run.smithi152.stdout:201863462951 2024-04-26T04:56:55.851 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1056667910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:56:55.852 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3389799173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:56:55.852 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1056667910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:56:55.852 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3389799173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:56:56.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1056667910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:56:56.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3389799173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:56:56.277 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:56.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:56 smithi152 ceph-mon[27560]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:56.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:56 smithi152 ceph-mon[39324]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:57.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:56 smithi195 ceph-mon[31427]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:58.036 INFO:teuthology.orchestra.run.smithi152.stdout:77309411386 2024-04-26T04:56:58.718 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:58 smithi152 ceph-mon[27560]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:58.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:56:58 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2274639978' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:56:58.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:58 smithi152 ceph-mon[39324]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:58.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:56:58 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2274639978' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:56:59.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:58 smithi195 ceph-mon[31427]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:59.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:56:58 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2274639978' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:56:59.440 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:59.465 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149745 got 111669149749 for osd.3 2024-04-26T04:56:59.465 DEBUG:teuthology.parallel:result is None 2024-04-26T04:56:59.862 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462948 got 201863462951 for osd.6 2024-04-26T04:56:59.863 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:00.828 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:00 smithi152 ceph-mon[27560]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:00.829 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:00 smithi152 ceph-mon[39324]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:01.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:00 smithi195 ceph-mon[31427]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:01.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:57:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:01.686 INFO:teuthology.orchestra.run.smithi152.stdout:55834574912 2024-04-26T04:57:02.053 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411386 for osd.2 2024-04-26T04:57:02.053 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:02.391 INFO:teuthology.orchestra.run.smithi152.stdout:34359738437 2024-04-26T04:57:02.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:02] "GET /metrics HTTP/1.1" 200 239059 "" "Prometheus/2.43.0" 2024-04-26T04:57:02.912 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:02 smithi152 ceph-mon[27560]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:02.912 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:02 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1675964789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:57:02.912 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:02 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1501737184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:57:02.912 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:02 smithi152 ceph-mon[39324]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:02.912 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:02 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1675964789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:57:02.912 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:02 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1501737184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:57:03.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:02 smithi195 ceph-mon[31427]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:03.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:02 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1675964789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:57:03.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:02 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1501737184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:57:03.343 INFO:teuthology.orchestra.run.smithi152.stdout:133143986225 2024-04-26T04:57:03.863 INFO:teuthology.orchestra.run.smithi152.stdout:236223201315 2024-04-26T04:57:03.863 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:03 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3872244900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:57:03.863 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:03 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3872244900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:57:04.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:03 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3872244900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:57:04.710 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986222 got 133143986225 for osd.4 2024-04-26T04:57:04.710 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:04.976 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:04 smithi152 ceph-mon[27560]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:04.976 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:04 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1914398132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:57:04.976 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:04 smithi152 ceph-mon[39324]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:04.976 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:04 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1914398132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:57:05.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:04 smithi195 ceph-mon[31427]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:05.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:04 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1914398132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:57:05.398 INFO:teuthology.orchestra.run.smithi152.stdout:167503724589 2024-04-26T04:57:06.017 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:05 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/314287684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:57:06.017 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:05 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/314287684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:57:06.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:05 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/314287684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:57:06.042 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724586 got 167503724589 for osd.5 2024-04-26T04:57:06.042 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:06.595 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738434 got 34359738437 for osd.0 2024-04-26T04:57:06.595 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:06.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:06 smithi152 ceph-mon[27560]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:06.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:06 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:06.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:06 smithi152 ceph-mon[39324]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:06.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:06 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:06.965 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574908 got 55834574912 for osd.1 2024-04-26T04:57:06.965 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:07.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:06 smithi195 ceph-mon[31427]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:07.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:06 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:07.461 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201315 for osd.7 2024-04-26T04:57:07.462 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:07.462 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:57:07.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:57:07.739 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:08.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:08 smithi152 ceph-mon[27560]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:08.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:08 smithi152 ceph-mon[39324]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:09.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:08 smithi195 ceph-mon[31427]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:09.645 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:57:09.646 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-26T04:57:10.377 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":83,"stamp":"2024-04-26T04:57:09.309041+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386340,"kb_used_data":7472,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574876,"statfs":{"total":767960285184,"available":765516673024,"internally_reserved":0,"allocated":7651328,"data_stored":4307481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620677+0000","last_change":"2024-04-26T04:56:08.422705+0000","last_active":"2024-04-26T04:56:51.620677+0000","last_peered":"2024-04-26T04:56:51.620677+0000","last_clean":"2024-04-26T04:56:51.620677+0000","last_became_active":"2024-04-26T04:56:08.422413+0000","last_became_peered":"2024-04-26T04:56:08.422413+0000","last_unstale":"2024-04-26T04:56:51.620677+0000","last_undegraded":"2024-04-26T04:56:51.620677+0000","last_fullsized":"2024-04-26T04:56:51.620677+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:41:08.837445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777052+0000","last_change":"2024-04-26T04:56:08.413358+0000","last_active":"2024-04-26T04:56:51.777052+0000","last_peered":"2024-04-26T04:56:51.777052+0000","last_clean":"2024-04-26T04:56:51.777052+0000","last_became_active":"2024-04-26T04:56:08.412626+0000","last_became_peered":"2024-04-26T04:56:08.412626+0000","last_unstale":"2024-04-26T04:56:51.777052+0000","last_undegraded":"2024-04-26T04:56:51.777052+0000","last_fullsized":"2024-04-26T04:56:51.777052+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:54:19.139776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617890+0000","last_change":"2024-04-26T04:56:09.590685+0000","last_active":"2024-04-26T04:56:51.617890+0000","last_peered":"2024-04-26T04:56:51.617890+0000","last_clean":"2024-04-26T04:56:51.617890+0000","last_became_active":"2024-04-26T04:56:09.590397+0000","last_became_peered":"2024-04-26T04:56:09.590397+0000","last_unstale":"2024-04-26T04:56:51.617890+0000","last_undegraded":"2024-04-26T04:56:51.617890+0000","last_fullsized":"2024-04-26T04:56:51.617890+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:27:08.966368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.618265+0000","last_change":"2024-04-26T04:56:08.405343+0000","last_active":"2024-04-26T04:56:51.618265+0000","last_peered":"2024-04-26T04:56:51.618265+0000","last_clean":"2024-04-26T04:56:51.618265+0000","last_became_active":"2024-04-26T04:56:08.404690+0000","last_became_peered":"2024-04-26T04:56:08.404690+0000","last_unstale":"2024-04-26T04:56:51.618265+0000","last_undegraded":"2024-04-26T04:56:51.618265+0000","last_fullsized":"2024-04-26T04:56:51.618265+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:11:22.349290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621836+0000","last_change":"2024-04-26T04:56:08.735659+0000","last_active":"2024-04-26T04:56:51.621836+0000","last_peered":"2024-04-26T04:56:51.621836+0000","last_clean":"2024-04-26T04:56:51.621836+0000","last_became_active":"2024-04-26T04:56:08.735340+0000","last_became_peered":"2024-04-26T04:56:08.735340+0000","last_unstale":"2024-04-26T04:56:51.621836+0000","last_undegraded":"2024-04-26T04:56:51.621836+0000","last_fullsized":"2024-04-26T04:56:51.621836+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:14:01.028490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617405+0000","last_change":"2024-04-26T04:56:08.400207+0000","last_active":"2024-04-26T04:56:51.617405+0000","last_peered":"2024-04-26T04:56:51.617405+0000","last_clean":"2024-04-26T04:56:51.617405+0000","last_became_active":"2024-04-26T04:56:08.399958+0000","last_became_peered":"2024-04-26T04:56:08.399958+0000","last_unstale":"2024-04-26T04:56:51.617405+0000","last_undegraded":"2024-04-26T04:56:51.617405+0000","last_fullsized":"2024-04-26T04:56:51.617405+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:58:07.756276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621369+0000","last_change":"2024-04-26T04:56:08.736247+0000","last_active":"2024-04-26T04:56:51.621369+0000","last_peered":"2024-04-26T04:56:51.621369+0000","last_clean":"2024-04-26T04:56:51.621369+0000","last_became_active":"2024-04-26T04:56:08.735982+0000","last_became_peered":"2024-04-26T04:56:08.735982+0000","last_unstale":"2024-04-26T04:56:51.621369+0000","last_undegraded":"2024-04-26T04:56:51.621369+0000","last_fullsized":"2024-04-26T04:56:51.621369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:47:18.541525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617296+0000","last_change":"2024-04-26T04:56:08.408106+0000","last_active":"2024-04-26T04:56:51.617296+0000","last_peered":"2024-04-26T04:56:51.617296+0000","last_clean":"2024-04-26T04:56:51.617296+0000","last_became_active":"2024-04-26T04:56:08.407842+0000","last_became_peered":"2024-04-26T04:56:08.407842+0000","last_unstale":"2024-04-26T04:56:51.617296+0000","last_undegraded":"2024-04-26T04:56:51.617296+0000","last_fullsized":"2024-04-26T04:56:51.617296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:24:40.677357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.619703+0000","last_change":"2024-04-26T04:56:08.735642+0000","last_active":"2024-04-26T04:56:51.619703+0000","last_peered":"2024-04-26T04:56:51.619703+0000","last_clean":"2024-04-26T04:56:51.619703+0000","last_became_active":"2024-04-26T04:56:08.735354+0000","last_became_peered":"2024-04-26T04:56:08.735354+0000","last_unstale":"2024-04-26T04:56:51.619703+0000","last_undegraded":"2024-04-26T04:56:51.619703+0000","last_fullsized":"2024-04-26T04:56:51.619703+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:33:01.887957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776795+0000","last_change":"2024-04-26T04:56:08.409101+0000","last_active":"2024-04-26T04:56:51.776795+0000","last_peered":"2024-04-26T04:56:51.776795+0000","last_clean":"2024-04-26T04:56:51.776795+0000","last_became_active":"2024-04-26T04:56:08.408607+0000","last_became_peered":"2024-04-26T04:56:08.408607+0000","last_unstale":"2024-04-26T04:56:51.776795+0000","last_undegraded":"2024-04-26T04:56:51.776795+0000","last_fullsized":"2024-04-26T04:56:51.776795+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:10:00.611740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621108+0000","last_change":"2024-04-26T04:56:08.417990+0000","last_active":"2024-04-26T04:56:51.621108+0000","last_peered":"2024-04-26T04:56:51.621108+0000","last_clean":"2024-04-26T04:56:51.621108+0000","last_became_active":"2024-04-26T04:56:08.417703+0000","last_became_peered":"2024-04-26T04:56:08.417703+0000","last_unstale":"2024-04-26T04:56:51.621108+0000","last_undegraded":"2024-04-26T04:56:51.621108+0000","last_fullsized":"2024-04-26T04:56:51.621108+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:49:52.041048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776915+0000","last_change":"2024-04-26T04:56:08.407508+0000","last_active":"2024-04-26T04:56:51.776915+0000","last_peered":"2024-04-26T04:56:51.776915+0000","last_clean":"2024-04-26T04:56:51.776915+0000","last_became_active":"2024-04-26T04:56:08.407199+0000","last_became_peered":"2024-04-26T04:56:08.407199+0000","last_unstale":"2024-04-26T04:56:51.776915+0000","last_undegraded":"2024-04-26T04:56:51.776915+0000","last_fullsized":"2024-04-26T04:56:51.776915+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:45:42.599189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620522+0000","last_change":"2024-04-26T04:54:55.373997+0000","last_active":"2024-04-26T04:56:51.620522+0000","last_peered":"2024-04-26T04:56:51.620522+0000","last_clean":"2024-04-26T04:56:51.620522+0000","last_became_active":"2024-04-26T04:54:55.373681+0000","last_became_peered":"2024-04-26T04:54:55.373681+0000","last_unstale":"2024-04-26T04:56:51.620522+0000","last_undegraded":"2024-04-26T04:56:51.620522+0000","last_fullsized":"2024-04-26T04:56:51.620522+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:10:19.893941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617502+0000","last_change":"2024-04-26T04:56:09.701337+0000","last_active":"2024-04-26T04:56:51.617502+0000","last_peered":"2024-04-26T04:56:51.617502+0000","last_clean":"2024-04-26T04:56:51.617502+0000","last_became_active":"2024-04-26T04:56:09.701024+0000","last_became_peered":"2024-04-26T04:56:09.701024+0000","last_unstale":"2024-04-26T04:56:51.617502+0000","last_undegraded":"2024-04-26T04:56:51.617502+0000","last_fullsized":"2024-04-26T04:56:51.617502+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:45:55.621201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620819+0000","last_change":"2024-04-26T04:54:55.567081+0000","last_active":"2024-04-26T04:56:51.620819+0000","last_peered":"2024-04-26T04:56:51.620819+0000","last_clean":"2024-04-26T04:56:51.620819+0000","last_became_active":"2024-04-26T04:54:55.566744+0000","last_became_peered":"2024-04-26T04:54:55.566744+0000","last_unstale":"2024-04-26T04:56:51.620819+0000","last_undegraded":"2024-04-26T04:56:51.620819+0000","last_fullsized":"2024-04-26T04:56:51.620819+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:46:08.438081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.618392+0000","last_change":"2024-04-26T04:56:08.415613+0000","last_active":"2024-04-26T04:56:51.618392+0000","last_peered":"2024-04-26T04:56:51.618392+0000","last_clean":"2024-04-26T04:56:51.618392+0000","last_became_active":"2024-04-26T04:56:08.415293+0000","last_became_peered":"2024-04-26T04:56:08.415293+0000","last_unstale":"2024-04-26T04:56:51.618392+0000","last_undegraded":"2024-04-26T04:56:51.618392+0000","last_fullsized":"2024-04-26T04:56:51.618392+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:40:51.684425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776830+0000","last_change":"2024-04-26T04:56:09.702411+0000","last_active":"2024-04-26T04:56:51.776830+0000","last_peered":"2024-04-26T04:56:51.776830+0000","last_clean":"2024-04-26T04:56:51.776830+0000","last_became_active":"2024-04-26T04:56:09.702123+0000","last_became_peered":"2024-04-26T04:56:09.702123+0000","last_unstale":"2024-04-26T04:56:51.776830+0000","last_undegraded":"2024-04-26T04:56:51.776830+0000","last_fullsized":"2024-04-26T04:56:51.776830+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:53.630964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620605+0000","last_change":"2024-04-26T04:56:08.414883+0000","last_active":"2024-04-26T04:56:51.620605+0000","last_peered":"2024-04-26T04:56:51.620605+0000","last_clean":"2024-04-26T04:56:51.620605+0000","last_became_active":"2024-04-26T04:56:08.412826+0000","last_became_peered":"2024-04-26T04:56:08.412826+0000","last_unstale":"2024-04-26T04:56:51.620605+0000","last_undegraded":"2024-04-26T04:56:51.620605+0000","last_fullsized":"2024-04-26T04:56:51.620605+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:14:31.003214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776958+0000","last_change":"2024-04-26T04:54:55.960283+0000","last_active":"2024-04-26T04:56:51.776958+0000","last_peered":"2024-04-26T04:56:51.776958+0000","last_clean":"2024-04-26T04:56:51.776958+0000","last_became_active":"2024-04-26T04:54:55.959970+0000","last_became_peered":"2024-04-26T04:54:55.959970+0000","last_unstale":"2024-04-26T04:56:51.776958+0000","last_undegraded":"2024-04-26T04:56:51.776958+0000","last_fullsized":"2024-04-26T04:56:51.776958+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:27:28.847430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776916+0000","last_change":"2024-04-26T04:56:09.590716+0000","last_active":"2024-04-26T04:56:51.776916+0000","last_peered":"2024-04-26T04:56:51.776916+0000","last_clean":"2024-04-26T04:56:51.776916+0000","last_became_active":"2024-04-26T04:56:09.590401+0000","last_became_peered":"2024-04-26T04:56:09.590401+0000","last_unstale":"2024-04-26T04:56:51.776916+0000","last_undegraded":"2024-04-26T04:56:51.776916+0000","last_fullsized":"2024-04-26T04:56:51.776916+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:43:55.376837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776802+0000","last_change":"2024-04-26T04:56:08.410196+0000","last_active":"2024-04-26T04:56:51.776802+0000","last_peered":"2024-04-26T04:56:51.776802+0000","last_clean":"2024-04-26T04:56:51.776802+0000","last_became_active":"2024-04-26T04:56:08.409945+0000","last_became_peered":"2024-04-26T04:56:08.409945+0000","last_unstale":"2024-04-26T04:56:51.776802+0000","last_undegraded":"2024-04-26T04:56:51.776802+0000","last_fullsized":"2024-04-26T04:56:51.776802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:34:41.434576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776689+0000","last_change":"2024-04-26T04:54:55.960282+0000","last_active":"2024-04-26T04:56:51.776689+0000","last_peered":"2024-04-26T04:56:51.776689+0000","last_clean":"2024-04-26T04:56:51.776689+0000","last_became_active":"2024-04-26T04:54:55.959964+0000","last_became_peered":"2024-04-26T04:54:55.959964+0000","last_unstale":"2024-04-26T04:56:51.776689+0000","last_undegraded":"2024-04-26T04:56:51.776689+0000","last_fullsized":"2024-04-26T04:56:51.776689+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:56:37.017502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620445+0000","last_change":"2024-04-26T04:56:08.735555+0000","last_active":"2024-04-26T04:56:51.620445+0000","last_peered":"2024-04-26T04:56:51.620445+0000","last_clean":"2024-04-26T04:56:51.620445+0000","last_became_active":"2024-04-26T04:56:08.735298+0000","last_became_peered":"2024-04-26T04:56:08.735298+0000","last_unstale":"2024-04-26T04:56:51.620445+0000","last_undegraded":"2024-04-26T04:56:51.620445+0000","last_fullsized":"2024-04-26T04:56:51.620445+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:08.329803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617719+0000","last_change":"2024-04-26T04:56:08.404581+0000","last_active":"2024-04-26T04:56:51.617719+0000","last_peered":"2024-04-26T04:56:51.617719+0000","last_clean":"2024-04-26T04:56:51.617719+0000","last_became_active":"2024-04-26T04:56:08.404293+0000","last_became_peered":"2024-04-26T04:56:08.404293+0000","last_unstale":"2024-04-26T04:56:51.617719+0000","last_undegraded":"2024-04-26T04:56:51.617719+0000","last_fullsized":"2024-04-26T04:56:51.617719+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:35:49.314242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776717+0000","last_change":"2024-04-26T04:54:24.066215+0000","last_active":"2024-04-26T04:56:51.776717+0000","last_peered":"2024-04-26T04:56:51.776717+0000","last_clean":"2024-04-26T04:56:51.776717+0000","last_became_active":"2024-04-26T04:54:24.065870+0000","last_became_peered":"2024-04-26T04:54:24.065870+0000","last_unstale":"2024-04-26T04:56:51.776717+0000","last_undegraded":"2024-04-26T04:56:51.776717+0000","last_fullsized":"2024-04-26T04:56:51.776717+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_clean_scrub_stamp":"2024-04-26T04:52:21.376988+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:15:38.049930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617101+0000","last_change":"2024-04-26T04:54:55.372814+0000","last_active":"2024-04-26T04:56:51.617101+0000","last_peered":"2024-04-26T04:56:51.617101+0000","last_clean":"2024-04-26T04:56:51.617101+0000","last_became_active":"2024-04-26T04:54:55.372511+0000","last_became_peered":"2024-04-26T04:54:55.372511+0000","last_unstale":"2024-04-26T04:56:51.617101+0000","last_undegraded":"2024-04-26T04:56:51.617101+0000","last_fullsized":"2024-04-26T04:56:51.617101+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:40.580236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620548+0000","last_change":"2024-04-26T04:56:09.417521+0000","last_active":"2024-04-26T04:56:51.620548+0000","last_peered":"2024-04-26T04:56:51.620548+0000","last_clean":"2024-04-26T04:56:51.620548+0000","last_became_active":"2024-04-26T04:56:09.417235+0000","last_became_peered":"2024-04-26T04:56:09.417235+0000","last_unstale":"2024-04-26T04:56:51.620548+0000","last_undegraded":"2024-04-26T04:56:51.620548+0000","last_fullsized":"2024-04-26T04:56:51.620548+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:20:45.955664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620895+0000","last_change":"2024-04-26T04:56:08.411074+0000","last_active":"2024-04-26T04:56:51.620895+0000","last_peered":"2024-04-26T04:56:51.620895+0000","last_clean":"2024-04-26T04:56:51.620895+0000","last_became_active":"2024-04-26T04:56:08.410747+0000","last_became_peered":"2024-04-26T04:56:08.410747+0000","last_unstale":"2024-04-26T04:56:51.620895+0000","last_undegraded":"2024-04-26T04:56:51.620895+0000","last_fullsized":"2024-04-26T04:56:51.620895+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:37:52.866142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.619564+0000","last_change":"2024-04-26T04:54:55.960600+0000","last_active":"2024-04-26T04:56:51.619564+0000","last_peered":"2024-04-26T04:56:51.619564+0000","last_clean":"2024-04-26T04:56:51.619564+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:56:51.619564+0000","last_undegraded":"2024-04-26T04:56:51.619564+0000","last_fullsized":"2024-04-26T04:56:51.619564+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:08:33.905027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620552+0000","last_change":"2024-04-26T04:56:09.590771+0000","last_active":"2024-04-26T04:56:51.620552+0000","last_peered":"2024-04-26T04:56:51.620552+0000","last_clean":"2024-04-26T04:56:51.620552+0000","last_became_active":"2024-04-26T04:56:09.590466+0000","last_became_peered":"2024-04-26T04:56:09.590466+0000","last_unstale":"2024-04-26T04:56:51.620552+0000","last_undegraded":"2024-04-26T04:56:51.620552+0000","last_fullsized":"2024-04-26T04:56:51.620552+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:58:13.135264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620639+0000","last_change":"2024-04-26T04:56:08.407163+0000","last_active":"2024-04-26T04:56:51.620639+0000","last_peered":"2024-04-26T04:56:51.620639+0000","last_clean":"2024-04-26T04:56:51.620639+0000","last_became_active":"2024-04-26T04:56:08.406919+0000","last_became_peered":"2024-04-26T04:56:08.406919+0000","last_unstale":"2024-04-26T04:56:51.620639+0000","last_undegraded":"2024-04-26T04:56:51.620639+0000","last_fullsized":"2024-04-26T04:56:51.620639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:19:56.376964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617000+0000","last_change":"2024-04-26T04:54:55.372743+0000","last_active":"2024-04-26T04:56:51.617000+0000","last_peered":"2024-04-26T04:56:51.617000+0000","last_clean":"2024-04-26T04:56:51.617000+0000","last_became_active":"2024-04-26T04:54:55.372440+0000","last_became_peered":"2024-04-26T04:54:55.372440+0000","last_unstale":"2024-04-26T04:56:51.617000+0000","last_undegraded":"2024-04-26T04:56:51.617000+0000","last_fullsized":"2024-04-26T04:56:51.617000+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:02:35.773050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:56:52.907901+0000","last_change":"2024-04-26T04:56:52.907901+0000","last_active":"2024-04-26T04:56:52.907901+0000","last_peered":"2024-04-26T04:56:52.907901+0000","last_clean":"2024-04-26T04:56:52.907901+0000","last_became_active":"2024-04-26T04:56:52.907625+0000","last_became_peered":"2024-04-26T04:56:52.907625+0000","last_unstale":"2024-04-26T04:56:52.907901+0000","last_undegraded":"2024-04-26T04:56:52.907901+0000","last_fullsized":"2024-04-26T04:56:52.907901+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:52:33.199158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617361+0000","last_change":"2024-04-26T04:56:08.409276+0000","last_active":"2024-04-26T04:56:51.617361+0000","last_peered":"2024-04-26T04:56:51.617361+0000","last_clean":"2024-04-26T04:56:51.617361+0000","last_became_active":"2024-04-26T04:56:08.409013+0000","last_became_peered":"2024-04-26T04:56:08.409013+0000","last_unstale":"2024-04-26T04:56:51.617361+0000","last_undegraded":"2024-04-26T04:56:51.617361+0000","last_fullsized":"2024-04-26T04:56:51.617361+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:36:59.616057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620479+0000","last_change":"2024-04-26T04:54:55.960319+0000","last_active":"2024-04-26T04:56:51.620479+0000","last_peered":"2024-04-26T04:56:51.620479+0000","last_clean":"2024-04-26T04:56:51.620479+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:56:51.620479+0000","last_undegraded":"2024-04-26T04:56:51.620479+0000","last_fullsized":"2024-04-26T04:56:51.620479+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:23:09.651998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620620+0000","last_change":"2024-04-26T04:56:09.703640+0000","last_active":"2024-04-26T04:56:51.620620+0000","last_peered":"2024-04-26T04:56:51.620620+0000","last_clean":"2024-04-26T04:56:51.620620+0000","last_became_active":"2024-04-26T04:56:09.703403+0000","last_became_peered":"2024-04-26T04:56:09.703403+0000","last_unstale":"2024-04-26T04:56:51.620620+0000","last_undegraded":"2024-04-26T04:56:51.620620+0000","last_fullsized":"2024-04-26T04:56:51.620620+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:46.948653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617504+0000","last_change":"2024-04-26T04:56:08.414956+0000","last_active":"2024-04-26T04:56:51.617504+0000","last_peered":"2024-04-26T04:56:51.617504+0000","last_clean":"2024-04-26T04:56:51.617504+0000","last_became_active":"2024-04-26T04:56:08.414690+0000","last_became_peered":"2024-04-26T04:56:08.414690+0000","last_unstale":"2024-04-26T04:56:51.617504+0000","last_undegraded":"2024-04-26T04:56:51.617504+0000","last_fullsized":"2024-04-26T04:56:51.617504+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:46:09.618427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621886+0000","last_change":"2024-04-26T04:56:08.422859+0000","last_active":"2024-04-26T04:56:51.621886+0000","last_peered":"2024-04-26T04:56:51.621886+0000","last_clean":"2024-04-26T04:56:51.621886+0000","last_became_active":"2024-04-26T04:56:08.422458+0000","last_became_peered":"2024-04-26T04:56:08.422458+0000","last_unstale":"2024-04-26T04:56:51.621886+0000","last_undegraded":"2024-04-26T04:56:51.621886+0000","last_fullsized":"2024-04-26T04:56:51.621886+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:14:22.398355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621200+0000","last_change":"2024-04-26T04:56:08.404428+0000","last_active":"2024-04-26T04:56:51.621200+0000","last_peered":"2024-04-26T04:56:51.621200+0000","last_clean":"2024-04-26T04:56:51.621200+0000","last_became_active":"2024-04-26T04:56:08.403880+0000","last_became_peered":"2024-04-26T04:56:08.403880+0000","last_unstale":"2024-04-26T04:56:51.621200+0000","last_undegraded":"2024-04-26T04:56:51.621200+0000","last_fullsized":"2024-04-26T04:56:51.621200+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:29:48.464556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621709+0000","last_change":"2024-04-26T04:56:08.422225+0000","last_active":"2024-04-26T04:56:51.621709+0000","last_peered":"2024-04-26T04:56:51.621709+0000","last_clean":"2024-04-26T04:56:51.621709+0000","last_became_active":"2024-04-26T04:56:08.421922+0000","last_became_peered":"2024-04-26T04:56:08.421922+0000","last_unstale":"2024-04-26T04:56:51.621709+0000","last_undegraded":"2024-04-26T04:56:51.621709+0000","last_fullsized":"2024-04-26T04:56:51.621709+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:41:25.502938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620610+0000","last_change":"2024-04-26T04:56:08.410384+0000","last_active":"2024-04-26T04:56:51.620610+0000","last_peered":"2024-04-26T04:56:51.620610+0000","last_clean":"2024-04-26T04:56:51.620610+0000","last_became_active":"2024-04-26T04:56:08.410076+0000","last_became_peered":"2024-04-26T04:56:08.410076+0000","last_unstale":"2024-04-26T04:56:51.620610+0000","last_undegraded":"2024-04-26T04:56:51.620610+0000","last_fullsized":"2024-04-26T04:56:51.620610+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:18:13.554446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621344+0000","last_change":"2024-04-26T04:56:08.736931+0000","last_active":"2024-04-26T04:56:51.621344+0000","last_peered":"2024-04-26T04:56:51.621344+0000","last_clean":"2024-04-26T04:56:51.621344+0000","last_became_active":"2024-04-26T04:56:08.736654+0000","last_became_peered":"2024-04-26T04:56:08.736654+0000","last_unstale":"2024-04-26T04:56:51.621344+0000","last_undegraded":"2024-04-26T04:56:51.621344+0000","last_fullsized":"2024-04-26T04:56:51.621344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:48.772903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617892+0000","last_change":"2024-04-26T04:56:08.414099+0000","last_active":"2024-04-26T04:56:51.617892+0000","last_peered":"2024-04-26T04:56:51.617892+0000","last_clean":"2024-04-26T04:56:51.617892+0000","last_became_active":"2024-04-26T04:56:08.413690+0000","last_became_peered":"2024-04-26T04:56:08.413690+0000","last_unstale":"2024-04-26T04:56:51.617892+0000","last_undegraded":"2024-04-26T04:56:51.617892+0000","last_fullsized":"2024-04-26T04:56:51.617892+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:39:48.905754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776681+0000","last_change":"2024-04-26T04:56:09.703515+0000","last_active":"2024-04-26T04:56:51.776681+0000","last_peered":"2024-04-26T04:56:51.776681+0000","last_clean":"2024-04-26T04:56:51.776681+0000","last_became_active":"2024-04-26T04:56:09.703229+0000","last_became_peered":"2024-04-26T04:56:09.703229+0000","last_unstale":"2024-04-26T04:56:51.776681+0000","last_undegraded":"2024-04-26T04:56:51.776681+0000","last_fullsized":"2024-04-26T04:56:51.776681+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:03:52.339898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617213+0000","last_change":"2024-04-26T04:56:08.408599+0000","last_active":"2024-04-26T04:56:51.617213+0000","last_peered":"2024-04-26T04:56:51.617213+0000","last_clean":"2024-04-26T04:56:51.617213+0000","last_became_active":"2024-04-26T04:56:08.408360+0000","last_became_peered":"2024-04-26T04:56:08.408360+0000","last_unstale":"2024-04-26T04:56:51.617213+0000","last_undegraded":"2024-04-26T04:56:51.617213+0000","last_fullsized":"2024-04-26T04:56:51.617213+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:01:42.788764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620407+0000","last_change":"2024-04-26T04:56:08.735526+0000","last_active":"2024-04-26T04:56:51.620407+0000","last_peered":"2024-04-26T04:56:51.620407+0000","last_clean":"2024-04-26T04:56:51.620407+0000","last_became_active":"2024-04-26T04:56:08.735232+0000","last_became_peered":"2024-04-26T04:56:08.735232+0000","last_unstale":"2024-04-26T04:56:51.620407+0000","last_undegraded":"2024-04-26T04:56:51.620407+0000","last_fullsized":"2024-04-26T04:56:51.620407+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:37:25.532847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777056+0000","last_change":"2024-04-26T04:56:08.400274+0000","last_active":"2024-04-26T04:56:51.777056+0000","last_peered":"2024-04-26T04:56:51.777056+0000","last_clean":"2024-04-26T04:56:51.777056+0000","last_became_active":"2024-04-26T04:56:08.400012+0000","last_became_peered":"2024-04-26T04:56:08.400012+0000","last_unstale":"2024-04-26T04:56:51.777056+0000","last_undegraded":"2024-04-26T04:56:51.777056+0000","last_fullsized":"2024-04-26T04:56:51.777056+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:06:01.413415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621027+0000","last_change":"2024-04-26T04:56:08.736920+0000","last_active":"2024-04-26T04:56:51.621027+0000","last_peered":"2024-04-26T04:56:51.621027+0000","last_clean":"2024-04-26T04:56:51.621027+0000","last_became_active":"2024-04-26T04:56:08.736599+0000","last_became_peered":"2024-04-26T04:56:08.736599+0000","last_unstale":"2024-04-26T04:56:51.621027+0000","last_undegraded":"2024-04-26T04:56:51.621027+0000","last_fullsized":"2024-04-26T04:56:51.621027+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:38:39.874495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777098+0000","last_change":"2024-04-26T04:56:08.407242+0000","last_active":"2024-04-26T04:56:51.777098+0000","last_peered":"2024-04-26T04:56:51.777098+0000","last_clean":"2024-04-26T04:56:51.777098+0000","last_became_active":"2024-04-26T04:56:08.406253+0000","last_became_peered":"2024-04-26T04:56:08.406253+0000","last_unstale":"2024-04-26T04:56:51.777098+0000","last_undegraded":"2024-04-26T04:56:51.777098+0000","last_fullsized":"2024-04-26T04:56:51.777098+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:10:24.314178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617247+0000","last_change":"2024-04-26T04:56:08.735825+0000","last_active":"2024-04-26T04:56:51.617247+0000","last_peered":"2024-04-26T04:56:51.617247+0000","last_clean":"2024-04-26T04:56:51.617247+0000","last_became_active":"2024-04-26T04:56:08.735226+0000","last_became_peered":"2024-04-26T04:56:08.735226+0000","last_unstale":"2024-04-26T04:56:51.617247+0000","last_undegraded":"2024-04-26T04:56:51.617247+0000","last_fullsized":"2024-04-26T04:56:51.617247+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:17:43.402733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777135+0000","last_change":"2024-04-26T04:56:08.407027+0000","last_active":"2024-04-26T04:56:51.777135+0000","last_peered":"2024-04-26T04:56:51.777135+0000","last_clean":"2024-04-26T04:56:51.777135+0000","last_became_active":"2024-04-26T04:56:08.406743+0000","last_became_peered":"2024-04-26T04:56:08.406743+0000","last_unstale":"2024-04-26T04:56:51.777135+0000","last_undegraded":"2024-04-26T04:56:51.777135+0000","last_fullsized":"2024-04-26T04:56:51.777135+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:59:50.242788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.618302+0000","last_change":"2024-04-26T04:56:09.703080+0000","last_active":"2024-04-26T04:56:51.618302+0000","last_peered":"2024-04-26T04:56:51.618302+0000","last_clean":"2024-04-26T04:56:51.618302+0000","last_became_active":"2024-04-26T04:56:09.702772+0000","last_became_peered":"2024-04-26T04:56:09.702772+0000","last_unstale":"2024-04-26T04:56:51.618302+0000","last_undegraded":"2024-04-26T04:56:51.618302+0000","last_fullsized":"2024-04-26T04:56:51.618302+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:09:10.245555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621591+0000","last_change":"2024-04-26T04:56:08.405481+0000","last_active":"2024-04-26T04:56:51.621591+0000","last_peered":"2024-04-26T04:56:51.621591+0000","last_clean":"2024-04-26T04:56:51.621591+0000","last_became_active":"2024-04-26T04:56:08.405217+0000","last_became_peered":"2024-04-26T04:56:08.405217+0000","last_unstale":"2024-04-26T04:56:51.621591+0000","last_undegraded":"2024-04-26T04:56:51.621591+0000","last_fullsized":"2024-04-26T04:56:51.621591+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:37:59.899823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776809+0000","last_change":"2024-04-26T04:56:09.702688+0000","last_active":"2024-04-26T04:56:51.776809+0000","last_peered":"2024-04-26T04:56:51.776809+0000","last_clean":"2024-04-26T04:56:51.776809+0000","last_became_active":"2024-04-26T04:56:09.702369+0000","last_became_peered":"2024-04-26T04:56:09.702369+0000","last_unstale":"2024-04-26T04:56:51.776809+0000","last_undegraded":"2024-04-26T04:56:51.776809+0000","last_fullsized":"2024-04-26T04:56:51.776809+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:46:02.167155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776639+0000","last_change":"2024-04-26T04:56:08.400160+0000","last_active":"2024-04-26T04:56:51.776639+0000","last_peered":"2024-04-26T04:56:51.776639+0000","last_clean":"2024-04-26T04:56:51.776639+0000","last_became_active":"2024-04-26T04:56:08.399849+0000","last_became_peered":"2024-04-26T04:56:08.399849+0000","last_unstale":"2024-04-26T04:56:51.776639+0000","last_undegraded":"2024-04-26T04:56:51.776639+0000","last_fullsized":"2024-04-26T04:56:51.776639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:26:13.567428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617178+0000","last_change":"2024-04-26T04:56:08.735832+0000","last_active":"2024-04-26T04:56:51.617178+0000","last_peered":"2024-04-26T04:56:51.617178+0000","last_clean":"2024-04-26T04:56:51.617178+0000","last_became_active":"2024-04-26T04:56:08.735579+0000","last_became_peered":"2024-04-26T04:56:08.735579+0000","last_unstale":"2024-04-26T04:56:51.617178+0000","last_undegraded":"2024-04-26T04:56:51.617178+0000","last_fullsized":"2024-04-26T04:56:51.617178+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:10:51.768335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776678+0000","last_change":"2024-04-26T04:56:08.418624+0000","last_active":"2024-04-26T04:56:51.776678+0000","last_peered":"2024-04-26T04:56:51.776678+0000","last_clean":"2024-04-26T04:56:51.776678+0000","last_became_active":"2024-04-26T04:56:08.418366+0000","last_became_peered":"2024-04-26T04:56:08.418366+0000","last_unstale":"2024-04-26T04:56:51.776678+0000","last_undegraded":"2024-04-26T04:56:51.776678+0000","last_fullsized":"2024-04-26T04:56:51.776678+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:34:57.765643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617949+0000","last_change":"2024-04-26T04:56:09.701457+0000","last_active":"2024-04-26T04:56:51.617949+0000","last_peered":"2024-04-26T04:56:51.617949+0000","last_clean":"2024-04-26T04:56:51.617949+0000","last_became_active":"2024-04-26T04:56:09.701132+0000","last_became_peered":"2024-04-26T04:56:09.701132+0000","last_unstale":"2024-04-26T04:56:51.617949+0000","last_undegraded":"2024-04-26T04:56:51.617949+0000","last_fullsized":"2024-04-26T04:56:51.617949+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:30:40.646109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776865+0000","last_change":"2024-04-26T04:56:08.405604+0000","last_active":"2024-04-26T04:56:51.776865+0000","last_peered":"2024-04-26T04:56:51.776865+0000","last_clean":"2024-04-26T04:56:51.776865+0000","last_became_active":"2024-04-26T04:56:08.405321+0000","last_became_peered":"2024-04-26T04:56:08.405321+0000","last_unstale":"2024-04-26T04:56:51.776865+0000","last_undegraded":"2024-04-26T04:56:51.776865+0000","last_fullsized":"2024-04-26T04:56:51.776865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:35:18.908328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620732+0000","last_change":"2024-04-26T04:56:08.421752+0000","last_active":"2024-04-26T04:56:51.620732+0000","last_peered":"2024-04-26T04:56:51.620732+0000","last_clean":"2024-04-26T04:56:51.620732+0000","last_became_active":"2024-04-26T04:56:08.421495+0000","last_became_peered":"2024-04-26T04:56:08.421495+0000","last_unstale":"2024-04-26T04:56:51.620732+0000","last_undegraded":"2024-04-26T04:56:51.620732+0000","last_fullsized":"2024-04-26T04:56:51.620732+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:31:54.194701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621445+0000","last_change":"2024-04-26T04:56:08.399546+0000","last_active":"2024-04-26T04:56:51.621445+0000","last_peered":"2024-04-26T04:56:51.621445+0000","last_clean":"2024-04-26T04:56:51.621445+0000","last_became_active":"2024-04-26T04:56:08.399266+0000","last_became_peered":"2024-04-26T04:56:08.399266+0000","last_unstale":"2024-04-26T04:56:51.621445+0000","last_undegraded":"2024-04-26T04:56:51.621445+0000","last_fullsized":"2024-04-26T04:56:51.621445+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:46:52.731455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620285+0000","last_change":"2024-04-26T04:56:09.702277+0000","last_active":"2024-04-26T04:56:51.620285+0000","last_peered":"2024-04-26T04:56:51.620285+0000","last_clean":"2024-04-26T04:56:51.620285+0000","last_became_active":"2024-04-26T04:56:09.701935+0000","last_became_peered":"2024-04-26T04:56:09.701935+0000","last_unstale":"2024-04-26T04:56:51.620285+0000","last_undegraded":"2024-04-26T04:56:51.620285+0000","last_fullsized":"2024-04-26T04:56:51.620285+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:55:40.135676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776949+0000","last_change":"2024-04-26T04:56:08.407185+0000","last_active":"2024-04-26T04:56:51.776949+0000","last_peered":"2024-04-26T04:56:51.776949+0000","last_clean":"2024-04-26T04:56:51.776949+0000","last_became_active":"2024-04-26T04:56:08.406921+0000","last_became_peered":"2024-04-26T04:56:08.406921+0000","last_unstale":"2024-04-26T04:56:51.776949+0000","last_undegraded":"2024-04-26T04:56:51.776949+0000","last_fullsized":"2024-04-26T04:56:51.776949+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:19:22.160857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617968+0000","last_change":"2024-04-26T04:56:09.702829+0000","last_active":"2024-04-26T04:56:51.617968+0000","last_peered":"2024-04-26T04:56:51.617968+0000","last_clean":"2024-04-26T04:56:51.617968+0000","last_became_active":"2024-04-26T04:56:09.702562+0000","last_became_peered":"2024-04-26T04:56:09.702562+0000","last_unstale":"2024-04-26T04:56:51.617968+0000","last_undegraded":"2024-04-26T04:56:51.617968+0000","last_fullsized":"2024-04-26T04:56:51.617968+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:00:30.433073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617436+0000","last_change":"2024-04-26T04:56:08.405882+0000","last_active":"2024-04-26T04:56:51.617436+0000","last_peered":"2024-04-26T04:56:51.617436+0000","last_clean":"2024-04-26T04:56:51.617436+0000","last_became_active":"2024-04-26T04:56:08.404791+0000","last_became_peered":"2024-04-26T04:56:08.404791+0000","last_unstale":"2024-04-26T04:56:51.617436+0000","last_undegraded":"2024-04-26T04:56:51.617436+0000","last_fullsized":"2024-04-26T04:56:51.617436+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:00:32.949944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:56:52.908129+0000","last_change":"2024-04-26T04:56:52.908129+0000","last_active":"2024-04-26T04:56:52.908129+0000","last_peered":"2024-04-26T04:56:52.908129+0000","last_clean":"2024-04-26T04:56:52.908129+0000","last_became_active":"2024-04-26T04:56:52.907768+0000","last_became_peered":"2024-04-26T04:56:52.907768+0000","last_unstale":"2024-04-26T04:56:52.908129+0000","last_undegraded":"2024-04-26T04:56:52.908129+0000","last_fullsized":"2024-04-26T04:56:52.908129+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:37:51.277368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777008+0000","last_change":"2024-04-26T04:56:08.417932+0000","last_active":"2024-04-26T04:56:51.777008+0000","last_peered":"2024-04-26T04:56:51.777008+0000","last_clean":"2024-04-26T04:56:51.777008+0000","last_became_active":"2024-04-26T04:56:08.417649+0000","last_became_peered":"2024-04-26T04:56:08.417649+0000","last_unstale":"2024-04-26T04:56:51.777008+0000","last_undegraded":"2024-04-26T04:56:51.777008+0000","last_fullsized":"2024-04-26T04:56:51.777008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:46:27.177515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620422+0000","last_change":"2024-04-26T04:56:08.423360+0000","last_active":"2024-04-26T04:56:51.620422+0000","last_peered":"2024-04-26T04:56:51.620422+0000","last_clean":"2024-04-26T04:56:51.620422+0000","last_became_active":"2024-04-26T04:56:08.423072+0000","last_became_peered":"2024-04-26T04:56:08.423072+0000","last_unstale":"2024-04-26T04:56:51.620422+0000","last_undegraded":"2024-04-26T04:56:51.620422+0000","last_fullsized":"2024-04-26T04:56:51.620422+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:18:04.178846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617791+0000","last_change":"2024-04-26T04:56:08.417723+0000","last_active":"2024-04-26T04:56:51.617791+0000","last_peered":"2024-04-26T04:56:51.617791+0000","last_clean":"2024-04-26T04:56:51.617791+0000","last_became_active":"2024-04-26T04:56:08.416629+0000","last_became_peered":"2024-04-26T04:56:08.416629+0000","last_unstale":"2024-04-26T04:56:51.617791+0000","last_undegraded":"2024-04-26T04:56:51.617791+0000","last_fullsized":"2024-04-26T04:56:51.617791+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:52:49.448467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776837+0000","last_change":"2024-04-26T04:56:09.702682+0000","last_active":"2024-04-26T04:56:51.776837+0000","last_peered":"2024-04-26T04:56:51.776837+0000","last_clean":"2024-04-26T04:56:51.776837+0000","last_became_active":"2024-04-26T04:56:09.702380+0000","last_became_peered":"2024-04-26T04:56:09.702380+0000","last_unstale":"2024-04-26T04:56:51.776837+0000","last_undegraded":"2024-04-26T04:56:51.776837+0000","last_fullsized":"2024-04-26T04:56:51.776837+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:56:31.820640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620330+0000","last_change":"2024-04-26T04:56:08.409816+0000","last_active":"2024-04-26T04:56:51.620330+0000","last_peered":"2024-04-26T04:56:51.620330+0000","last_clean":"2024-04-26T04:56:51.620330+0000","last_became_active":"2024-04-26T04:56:08.409544+0000","last_became_peered":"2024-04-26T04:56:08.409544+0000","last_unstale":"2024-04-26T04:56:51.620330+0000","last_undegraded":"2024-04-26T04:56:51.620330+0000","last_fullsized":"2024-04-26T04:56:51.620330+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:51:16.478567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:56:52.857921+0000","last_change":"2024-04-26T04:56:52.857921+0000","last_active":"2024-04-26T04:56:52.857921+0000","last_peered":"2024-04-26T04:56:52.857921+0000","last_clean":"2024-04-26T04:56:52.857921+0000","last_became_active":"2024-04-26T04:56:52.857664+0000","last_became_peered":"2024-04-26T04:56:52.857664+0000","last_unstale":"2024-04-26T04:56:52.857921+0000","last_undegraded":"2024-04-26T04:56:52.857921+0000","last_fullsized":"2024-04-26T04:56:52.857921+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:11:11.311316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621751+0000","last_change":"2024-04-26T04:56:08.405077+0000","last_active":"2024-04-26T04:56:51.621751+0000","last_peered":"2024-04-26T04:56:51.621751+0000","last_clean":"2024-04-26T04:56:51.621751+0000","last_became_active":"2024-04-26T04:56:08.404805+0000","last_became_peered":"2024-04-26T04:56:08.404805+0000","last_unstale":"2024-04-26T04:56:51.621751+0000","last_undegraded":"2024-04-26T04:56:51.621751+0000","last_fullsized":"2024-04-26T04:56:51.621751+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:16:41.050046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.504,"5min":0.499,"15min":0.498},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.68500000000000005}]},{"osd":1,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51500000000000001,"15min":0.51300000000000001},"min":{"1min":0.318,"5min":0.316,"15min":0.316},"max":{"1min":0.77500000000000002,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.58599999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.80400000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.70199999999999996}]},{"osd":3,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53100000000000003,"15min":0.53000000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56399999999999995,"15min":0.56299999999999994},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.57699999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.48399999999999999,"15min":0.47599999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.79200000000000004,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.45900000000000002,"15min":0.44900000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73099999999999998,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.496}]},{"osd":5,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.47099999999999997,"15min":0.46000000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.84399999999999997,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.50900000000000001,"15min":0.501},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.753,"5min":0.878,"15min":0.878},"last":0.55400000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.503,"15min":0.495},"min":{"1min":0.35899999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.85899999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.55300000000000005,"15min":0.54400000000000004},"min":{"1min":0.38600000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.82999999999999996,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.622}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1300,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1331200,"data_stored":908128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.41799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69599999999999995,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51600000000000001,"15min":0.503},"min":{"1min":0.438,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.64400000000000002,"5min":0.745,"15min":0.745},"last":0.53700000000000003}]},{"osd":1,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.43099999999999999,"5min":0.38,"15min":0.38},"max":{"1min":0.755,"5min":1.008,"15min":1.008},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55100000000000005,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72399999999999998,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.57899999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.39800000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.625,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54500000000000004,"15min":0.55000000000000004},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77900000000000003,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.55200000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53000000000000003,"15min":0.52700000000000002},"min":{"1min":0.40100000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.66100000000000003,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38400000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.63800000000000001,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.56499999999999995}]},{"osd":4,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44500000000000001,"15min":0.439},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.73099999999999998,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.45300000000000001,"15min":0.442},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66800000000000004,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.63400000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68700000000000006,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.48699999999999999,"15min":0.47999999999999998},"min":{"1min":0.31900000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.70499999999999996,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.59499999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.48499999999999999,"15min":0.47699999999999998},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.65300000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":315931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:57:02 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.35199999999999998,"15min":0.33300000000000002},"min":{"1min":0.222,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.52100000000000002,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.36799999999999999,"15min":0.33400000000000002},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.495,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.34999999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.27800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.56799999999999995,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.53900000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.436}]},{"osd":3,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42399999999999999,"15min":0.436},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.499,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.41299999999999998,"15min":0.42899999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.44700000000000001,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.44700000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:56:09 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.41299999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.44600000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.67300000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:56:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.37,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58099999999999996,"15min":0.58399999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.64800000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:57:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.40000000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.621},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82399999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.69499999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57399999999999995,"15min":0.56599999999999995},"min":{"1min":0.42099999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.57299999999999995,"15min":0.56299999999999994},"min":{"1min":0.42299999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.61099999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":907421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.38200000000000001,"15min":0.34200000000000003},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.62,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.374,"15min":0.33000000000000002},"min":{"1min":0.251,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.38600000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.623,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.41699999999999998,"15min":0.40899999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.56499999999999995,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40600000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.44500000000000001,"15min":0.46899999999999997},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56999999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.41499999999999998,"15min":0.433},"min":{"1min":0.26600000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55800000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.36599999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.622,"15min":0.63900000000000001},"min":{"1min":0.41999999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.623,"15min":0.63700000000000001},"min":{"1min":0.42999999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.621}]},{"osd":5,"last update":"Fri Apr 26 04:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59399999999999997,"15min":0.59599999999999997},"min":{"1min":0.45000000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.84199999999999997,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.40100000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76400000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.58799999999999997,"15min":0.58899999999999997},"min":{"1min":0.41999999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.76100000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.44900000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.60399999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.55400000000000005,"15min":0.54700000000000004},"min":{"1min":0.45900000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.58299999999999996,"15min":0.57499999999999996},"min":{"1min":0.42799999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.64100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.34499999999999997,"15min":0.33800000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51800000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.379},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.36199999999999999,"15min":0.35999999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.54200000000000004,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.52300000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.35799999999999998,"15min":0.35699999999999998},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.40400000000000003,"15min":0.39600000000000002},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56200000000000006,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.441}]},{"osd":3,"last update":"Fri Apr 26 04:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45900000000000002,"15min":0.46700000000000003},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.44,"15min":0.45200000000000001},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.496}]},{"osd":4,"last update":"Fri Apr 26 04:56:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.56799999999999995,"15min":0.56000000000000005},"min":{"1min":0.44,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.74099999999999999,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.754,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.64600000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:56:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54900000000000004,"15min":0.54100000000000004},"min":{"1min":0.39400000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57799999999999996,"15min":0.58299999999999996},"min":{"1min":0.44400000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.60799999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:57:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60299999999999998,"15min":0.61099999999999999},"min":{"1min":0.42999999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81200000000000006,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.59999999999999998,"15min":0.61399999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84599999999999997,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.59799999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76800000000000002,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.63100000000000001,"15min":0.63600000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.81599999999999995,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.63300000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.38100000000000001,"15min":0.39400000000000002},"min":{"1min":0.253,"5min":0.245,"15min":0.245},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41699999999999998,"15min":0.43099999999999999},"min":{"1min":0.308,"5min":0.245,"15min":0.245},"max":{"1min":0.54700000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.33800000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.40600000000000003,"15min":0.40799999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.53700000000000003,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.40699999999999997,"15min":0.42199999999999999},"min":{"1min":0.27900000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.57199999999999995,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.32200000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.441,"15min":0.45400000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.66100000000000003,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42999999999999999,"15min":0.45200000000000001},"min":{"1min":0.28699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.35199999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:56:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.70599999999999996}]},{"osd":5,"last update":"Fri Apr 26 04:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.57899999999999996,"15min":0.59499999999999997},"min":{"1min":0.38500000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.77000000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.873},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.40300000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.60499999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:57:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58999999999999997,"15min":0.59499999999999997},"min":{"1min":0.44800000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.77700000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:56:22 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58399999999999996,"15min":0.57999999999999996},"min":{"1min":0.46100000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.754},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61599999999999999,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.82599999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":745472,"data_stored":317894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.38200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69999999999999996,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.53800000000000003},"min":{"1min":0.41299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.70199999999999996,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.66600000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.55900000000000005},"min":{"1min":0.36499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61799999999999999,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55600000000000005,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.71699999999999997,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.68600000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.68400000000000005,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.44500000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.68200000000000005,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.61499999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.56699999999999995,"15min":0.55300000000000005},"min":{"1min":0.44,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73599999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59299999999999997,"15min":0.60099999999999998},"min":{"1min":0.44500000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.754,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.69799999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:56:39 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.40999999999999998,"15min":0.39400000000000002},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.56399999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.42999999999999999,"15min":0.40999999999999998},"min":{"1min":0.32600000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.61399999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.34699999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:57:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47699999999999998,"15min":0.47499999999999998},"min":{"1min":0.34200000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66800000000000004,"5min":0.755,"15min":0.755},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.47699999999999998,"15min":0.46999999999999997},"min":{"1min":0.34499999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.69599999999999995,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.46500000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:56:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.40699999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.443},{"interface":"front","average":{"1min":0.501,"5min":0.51600000000000001,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.48599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.29199999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.68100000000000005,"5min":0.748,"15min":0.748},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.66600000000000004,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.443}]},{"osd":1,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53900000000000003,"15min":0.53500000000000003},"min":{"1min":0.36399999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67700000000000005,"5min":0.753,"15min":0.753},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.503,"15min":0.503},"min":{"1min":0.38300000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.65300000000000002,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.40999999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55100000000000005,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71099999999999997,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70199999999999996,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.65600000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67300000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56299999999999994,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.68500000000000005,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.495}]},{"osd":4,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.40200000000000002,"15min":0.39000000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.58599999999999997,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.42899999999999999,"15min":0.40699999999999997},"min":{"1min":0.32500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64200000000000002,"5min":0.75,"15min":0.75},"last":0.54200000000000004}]},{"osd":6,"last update":"Fri Apr 26 04:56:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.45500000000000002,"15min":0.42599999999999999},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.46300000000000002,"15min":0.438},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:56:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.38700000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.76500000000000001,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.624},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.34100000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.63900000000000001,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52600000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:57:10.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:57:10.636 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:10.916 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:10 smithi152 ceph-mon[27560]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:10.916 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:10 smithi152 ceph-mon[27560]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:57:10.916 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:10 smithi152 ceph-mon[39324]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:10.916 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:10 smithi152 ceph-mon[39324]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:57:11.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:57:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:11.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:10 smithi195 ceph-mon[31427]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:11.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:10 smithi195 ceph-mon[31427]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:57:12.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:12 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:12] "GET /metrics HTTP/1.1" 200 239123 "" "Prometheus/2.43.0" 2024-04-26T04:57:12.695 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:57:12.696 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-26T04:57:13.019 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:12 smithi152 ceph-mon[27560]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:13.019 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:12 smithi152 ceph-mon[39324]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:13.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:12 smithi195 ceph-mon[31427]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:13.472 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":84,"stamp":"2024-04-26T04:57:11.309404+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386340,"kb_used_data":7472,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574876,"statfs":{"total":767960285184,"available":765516673024,"internally_reserved":0,"allocated":7651328,"data_stored":4307481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002450"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620677+0000","last_change":"2024-04-26T04:56:08.422705+0000","last_active":"2024-04-26T04:56:51.620677+0000","last_peered":"2024-04-26T04:56:51.620677+0000","last_clean":"2024-04-26T04:56:51.620677+0000","last_became_active":"2024-04-26T04:56:08.422413+0000","last_became_peered":"2024-04-26T04:56:08.422413+0000","last_unstale":"2024-04-26T04:56:51.620677+0000","last_undegraded":"2024-04-26T04:56:51.620677+0000","last_fullsized":"2024-04-26T04:56:51.620677+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:41:08.837445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777052+0000","last_change":"2024-04-26T04:56:08.413358+0000","last_active":"2024-04-26T04:56:51.777052+0000","last_peered":"2024-04-26T04:56:51.777052+0000","last_clean":"2024-04-26T04:56:51.777052+0000","last_became_active":"2024-04-26T04:56:08.412626+0000","last_became_peered":"2024-04-26T04:56:08.412626+0000","last_unstale":"2024-04-26T04:56:51.777052+0000","last_undegraded":"2024-04-26T04:56:51.777052+0000","last_fullsized":"2024-04-26T04:56:51.777052+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:54:19.139776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617890+0000","last_change":"2024-04-26T04:56:09.590685+0000","last_active":"2024-04-26T04:56:51.617890+0000","last_peered":"2024-04-26T04:56:51.617890+0000","last_clean":"2024-04-26T04:56:51.617890+0000","last_became_active":"2024-04-26T04:56:09.590397+0000","last_became_peered":"2024-04-26T04:56:09.590397+0000","last_unstale":"2024-04-26T04:56:51.617890+0000","last_undegraded":"2024-04-26T04:56:51.617890+0000","last_fullsized":"2024-04-26T04:56:51.617890+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:27:08.966368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.618265+0000","last_change":"2024-04-26T04:56:08.405343+0000","last_active":"2024-04-26T04:56:51.618265+0000","last_peered":"2024-04-26T04:56:51.618265+0000","last_clean":"2024-04-26T04:56:51.618265+0000","last_became_active":"2024-04-26T04:56:08.404690+0000","last_became_peered":"2024-04-26T04:56:08.404690+0000","last_unstale":"2024-04-26T04:56:51.618265+0000","last_undegraded":"2024-04-26T04:56:51.618265+0000","last_fullsized":"2024-04-26T04:56:51.618265+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:11:22.349290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621836+0000","last_change":"2024-04-26T04:56:08.735659+0000","last_active":"2024-04-26T04:56:51.621836+0000","last_peered":"2024-04-26T04:56:51.621836+0000","last_clean":"2024-04-26T04:56:51.621836+0000","last_became_active":"2024-04-26T04:56:08.735340+0000","last_became_peered":"2024-04-26T04:56:08.735340+0000","last_unstale":"2024-04-26T04:56:51.621836+0000","last_undegraded":"2024-04-26T04:56:51.621836+0000","last_fullsized":"2024-04-26T04:56:51.621836+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:14:01.028490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617405+0000","last_change":"2024-04-26T04:56:08.400207+0000","last_active":"2024-04-26T04:56:51.617405+0000","last_peered":"2024-04-26T04:56:51.617405+0000","last_clean":"2024-04-26T04:56:51.617405+0000","last_became_active":"2024-04-26T04:56:08.399958+0000","last_became_peered":"2024-04-26T04:56:08.399958+0000","last_unstale":"2024-04-26T04:56:51.617405+0000","last_undegraded":"2024-04-26T04:56:51.617405+0000","last_fullsized":"2024-04-26T04:56:51.617405+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:58:07.756276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621369+0000","last_change":"2024-04-26T04:56:08.736247+0000","last_active":"2024-04-26T04:56:51.621369+0000","last_peered":"2024-04-26T04:56:51.621369+0000","last_clean":"2024-04-26T04:56:51.621369+0000","last_became_active":"2024-04-26T04:56:08.735982+0000","last_became_peered":"2024-04-26T04:56:08.735982+0000","last_unstale":"2024-04-26T04:56:51.621369+0000","last_undegraded":"2024-04-26T04:56:51.621369+0000","last_fullsized":"2024-04-26T04:56:51.621369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:47:18.541525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617296+0000","last_change":"2024-04-26T04:56:08.408106+0000","last_active":"2024-04-26T04:56:51.617296+0000","last_peered":"2024-04-26T04:56:51.617296+0000","last_clean":"2024-04-26T04:56:51.617296+0000","last_became_active":"2024-04-26T04:56:08.407842+0000","last_became_peered":"2024-04-26T04:56:08.407842+0000","last_unstale":"2024-04-26T04:56:51.617296+0000","last_undegraded":"2024-04-26T04:56:51.617296+0000","last_fullsized":"2024-04-26T04:56:51.617296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:24:40.677357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.619703+0000","last_change":"2024-04-26T04:56:08.735642+0000","last_active":"2024-04-26T04:56:51.619703+0000","last_peered":"2024-04-26T04:56:51.619703+0000","last_clean":"2024-04-26T04:56:51.619703+0000","last_became_active":"2024-04-26T04:56:08.735354+0000","last_became_peered":"2024-04-26T04:56:08.735354+0000","last_unstale":"2024-04-26T04:56:51.619703+0000","last_undegraded":"2024-04-26T04:56:51.619703+0000","last_fullsized":"2024-04-26T04:56:51.619703+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:33:01.887957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776795+0000","last_change":"2024-04-26T04:56:08.409101+0000","last_active":"2024-04-26T04:56:51.776795+0000","last_peered":"2024-04-26T04:56:51.776795+0000","last_clean":"2024-04-26T04:56:51.776795+0000","last_became_active":"2024-04-26T04:56:08.408607+0000","last_became_peered":"2024-04-26T04:56:08.408607+0000","last_unstale":"2024-04-26T04:56:51.776795+0000","last_undegraded":"2024-04-26T04:56:51.776795+0000","last_fullsized":"2024-04-26T04:56:51.776795+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:10:00.611740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621108+0000","last_change":"2024-04-26T04:56:08.417990+0000","last_active":"2024-04-26T04:56:51.621108+0000","last_peered":"2024-04-26T04:56:51.621108+0000","last_clean":"2024-04-26T04:56:51.621108+0000","last_became_active":"2024-04-26T04:56:08.417703+0000","last_became_peered":"2024-04-26T04:56:08.417703+0000","last_unstale":"2024-04-26T04:56:51.621108+0000","last_undegraded":"2024-04-26T04:56:51.621108+0000","last_fullsized":"2024-04-26T04:56:51.621108+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:49:52.041048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776915+0000","last_change":"2024-04-26T04:56:08.407508+0000","last_active":"2024-04-26T04:56:51.776915+0000","last_peered":"2024-04-26T04:56:51.776915+0000","last_clean":"2024-04-26T04:56:51.776915+0000","last_became_active":"2024-04-26T04:56:08.407199+0000","last_became_peered":"2024-04-26T04:56:08.407199+0000","last_unstale":"2024-04-26T04:56:51.776915+0000","last_undegraded":"2024-04-26T04:56:51.776915+0000","last_fullsized":"2024-04-26T04:56:51.776915+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:45:42.599189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620522+0000","last_change":"2024-04-26T04:54:55.373997+0000","last_active":"2024-04-26T04:56:51.620522+0000","last_peered":"2024-04-26T04:56:51.620522+0000","last_clean":"2024-04-26T04:56:51.620522+0000","last_became_active":"2024-04-26T04:54:55.373681+0000","last_became_peered":"2024-04-26T04:54:55.373681+0000","last_unstale":"2024-04-26T04:56:51.620522+0000","last_undegraded":"2024-04-26T04:56:51.620522+0000","last_fullsized":"2024-04-26T04:56:51.620522+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:10:19.893941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617502+0000","last_change":"2024-04-26T04:56:09.701337+0000","last_active":"2024-04-26T04:56:51.617502+0000","last_peered":"2024-04-26T04:56:51.617502+0000","last_clean":"2024-04-26T04:56:51.617502+0000","last_became_active":"2024-04-26T04:56:09.701024+0000","last_became_peered":"2024-04-26T04:56:09.701024+0000","last_unstale":"2024-04-26T04:56:51.617502+0000","last_undegraded":"2024-04-26T04:56:51.617502+0000","last_fullsized":"2024-04-26T04:56:51.617502+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:45:55.621201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620819+0000","last_change":"2024-04-26T04:54:55.567081+0000","last_active":"2024-04-26T04:56:51.620819+0000","last_peered":"2024-04-26T04:56:51.620819+0000","last_clean":"2024-04-26T04:56:51.620819+0000","last_became_active":"2024-04-26T04:54:55.566744+0000","last_became_peered":"2024-04-26T04:54:55.566744+0000","last_unstale":"2024-04-26T04:56:51.620819+0000","last_undegraded":"2024-04-26T04:56:51.620819+0000","last_fullsized":"2024-04-26T04:56:51.620819+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:46:08.438081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.618392+0000","last_change":"2024-04-26T04:56:08.415613+0000","last_active":"2024-04-26T04:56:51.618392+0000","last_peered":"2024-04-26T04:56:51.618392+0000","last_clean":"2024-04-26T04:56:51.618392+0000","last_became_active":"2024-04-26T04:56:08.415293+0000","last_became_peered":"2024-04-26T04:56:08.415293+0000","last_unstale":"2024-04-26T04:56:51.618392+0000","last_undegraded":"2024-04-26T04:56:51.618392+0000","last_fullsized":"2024-04-26T04:56:51.618392+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:40:51.684425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776830+0000","last_change":"2024-04-26T04:56:09.702411+0000","last_active":"2024-04-26T04:56:51.776830+0000","last_peered":"2024-04-26T04:56:51.776830+0000","last_clean":"2024-04-26T04:56:51.776830+0000","last_became_active":"2024-04-26T04:56:09.702123+0000","last_became_peered":"2024-04-26T04:56:09.702123+0000","last_unstale":"2024-04-26T04:56:51.776830+0000","last_undegraded":"2024-04-26T04:56:51.776830+0000","last_fullsized":"2024-04-26T04:56:51.776830+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:53.630964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620605+0000","last_change":"2024-04-26T04:56:08.414883+0000","last_active":"2024-04-26T04:56:51.620605+0000","last_peered":"2024-04-26T04:56:51.620605+0000","last_clean":"2024-04-26T04:56:51.620605+0000","last_became_active":"2024-04-26T04:56:08.412826+0000","last_became_peered":"2024-04-26T04:56:08.412826+0000","last_unstale":"2024-04-26T04:56:51.620605+0000","last_undegraded":"2024-04-26T04:56:51.620605+0000","last_fullsized":"2024-04-26T04:56:51.620605+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:14:31.003214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776958+0000","last_change":"2024-04-26T04:54:55.960283+0000","last_active":"2024-04-26T04:56:51.776958+0000","last_peered":"2024-04-26T04:56:51.776958+0000","last_clean":"2024-04-26T04:56:51.776958+0000","last_became_active":"2024-04-26T04:54:55.959970+0000","last_became_peered":"2024-04-26T04:54:55.959970+0000","last_unstale":"2024-04-26T04:56:51.776958+0000","last_undegraded":"2024-04-26T04:56:51.776958+0000","last_fullsized":"2024-04-26T04:56:51.776958+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:27:28.847430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776916+0000","last_change":"2024-04-26T04:56:09.590716+0000","last_active":"2024-04-26T04:56:51.776916+0000","last_peered":"2024-04-26T04:56:51.776916+0000","last_clean":"2024-04-26T04:56:51.776916+0000","last_became_active":"2024-04-26T04:56:09.590401+0000","last_became_peered":"2024-04-26T04:56:09.590401+0000","last_unstale":"2024-04-26T04:56:51.776916+0000","last_undegraded":"2024-04-26T04:56:51.776916+0000","last_fullsized":"2024-04-26T04:56:51.776916+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:43:55.376837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776802+0000","last_change":"2024-04-26T04:56:08.410196+0000","last_active":"2024-04-26T04:56:51.776802+0000","last_peered":"2024-04-26T04:56:51.776802+0000","last_clean":"2024-04-26T04:56:51.776802+0000","last_became_active":"2024-04-26T04:56:08.409945+0000","last_became_peered":"2024-04-26T04:56:08.409945+0000","last_unstale":"2024-04-26T04:56:51.776802+0000","last_undegraded":"2024-04-26T04:56:51.776802+0000","last_fullsized":"2024-04-26T04:56:51.776802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:34:41.434576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776689+0000","last_change":"2024-04-26T04:54:55.960282+0000","last_active":"2024-04-26T04:56:51.776689+0000","last_peered":"2024-04-26T04:56:51.776689+0000","last_clean":"2024-04-26T04:56:51.776689+0000","last_became_active":"2024-04-26T04:54:55.959964+0000","last_became_peered":"2024-04-26T04:54:55.959964+0000","last_unstale":"2024-04-26T04:56:51.776689+0000","last_undegraded":"2024-04-26T04:56:51.776689+0000","last_fullsized":"2024-04-26T04:56:51.776689+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:56:37.017502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620445+0000","last_change":"2024-04-26T04:56:08.735555+0000","last_active":"2024-04-26T04:56:51.620445+0000","last_peered":"2024-04-26T04:56:51.620445+0000","last_clean":"2024-04-26T04:56:51.620445+0000","last_became_active":"2024-04-26T04:56:08.735298+0000","last_became_peered":"2024-04-26T04:56:08.735298+0000","last_unstale":"2024-04-26T04:56:51.620445+0000","last_undegraded":"2024-04-26T04:56:51.620445+0000","last_fullsized":"2024-04-26T04:56:51.620445+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:08.329803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617719+0000","last_change":"2024-04-26T04:56:08.404581+0000","last_active":"2024-04-26T04:56:51.617719+0000","last_peered":"2024-04-26T04:56:51.617719+0000","last_clean":"2024-04-26T04:56:51.617719+0000","last_became_active":"2024-04-26T04:56:08.404293+0000","last_became_peered":"2024-04-26T04:56:08.404293+0000","last_unstale":"2024-04-26T04:56:51.617719+0000","last_undegraded":"2024-04-26T04:56:51.617719+0000","last_fullsized":"2024-04-26T04:56:51.617719+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:35:49.314242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776717+0000","last_change":"2024-04-26T04:54:24.066215+0000","last_active":"2024-04-26T04:56:51.776717+0000","last_peered":"2024-04-26T04:56:51.776717+0000","last_clean":"2024-04-26T04:56:51.776717+0000","last_became_active":"2024-04-26T04:54:24.065870+0000","last_became_peered":"2024-04-26T04:54:24.065870+0000","last_unstale":"2024-04-26T04:56:51.776717+0000","last_undegraded":"2024-04-26T04:56:51.776717+0000","last_fullsized":"2024-04-26T04:56:51.776717+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_clean_scrub_stamp":"2024-04-26T04:52:21.376988+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:15:38.049930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617101+0000","last_change":"2024-04-26T04:54:55.372814+0000","last_active":"2024-04-26T04:56:51.617101+0000","last_peered":"2024-04-26T04:56:51.617101+0000","last_clean":"2024-04-26T04:56:51.617101+0000","last_became_active":"2024-04-26T04:54:55.372511+0000","last_became_peered":"2024-04-26T04:54:55.372511+0000","last_unstale":"2024-04-26T04:56:51.617101+0000","last_undegraded":"2024-04-26T04:56:51.617101+0000","last_fullsized":"2024-04-26T04:56:51.617101+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:40.580236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620548+0000","last_change":"2024-04-26T04:56:09.417521+0000","last_active":"2024-04-26T04:56:51.620548+0000","last_peered":"2024-04-26T04:56:51.620548+0000","last_clean":"2024-04-26T04:56:51.620548+0000","last_became_active":"2024-04-26T04:56:09.417235+0000","last_became_peered":"2024-04-26T04:56:09.417235+0000","last_unstale":"2024-04-26T04:56:51.620548+0000","last_undegraded":"2024-04-26T04:56:51.620548+0000","last_fullsized":"2024-04-26T04:56:51.620548+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:20:45.955664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620895+0000","last_change":"2024-04-26T04:56:08.411074+0000","last_active":"2024-04-26T04:56:51.620895+0000","last_peered":"2024-04-26T04:56:51.620895+0000","last_clean":"2024-04-26T04:56:51.620895+0000","last_became_active":"2024-04-26T04:56:08.410747+0000","last_became_peered":"2024-04-26T04:56:08.410747+0000","last_unstale":"2024-04-26T04:56:51.620895+0000","last_undegraded":"2024-04-26T04:56:51.620895+0000","last_fullsized":"2024-04-26T04:56:51.620895+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:37:52.866142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.619564+0000","last_change":"2024-04-26T04:54:55.960600+0000","last_active":"2024-04-26T04:56:51.619564+0000","last_peered":"2024-04-26T04:56:51.619564+0000","last_clean":"2024-04-26T04:56:51.619564+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:56:51.619564+0000","last_undegraded":"2024-04-26T04:56:51.619564+0000","last_fullsized":"2024-04-26T04:56:51.619564+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:08:33.905027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620552+0000","last_change":"2024-04-26T04:56:09.590771+0000","last_active":"2024-04-26T04:56:51.620552+0000","last_peered":"2024-04-26T04:56:51.620552+0000","last_clean":"2024-04-26T04:56:51.620552+0000","last_became_active":"2024-04-26T04:56:09.590466+0000","last_became_peered":"2024-04-26T04:56:09.590466+0000","last_unstale":"2024-04-26T04:56:51.620552+0000","last_undegraded":"2024-04-26T04:56:51.620552+0000","last_fullsized":"2024-04-26T04:56:51.620552+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:58:13.135264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620639+0000","last_change":"2024-04-26T04:56:08.407163+0000","last_active":"2024-04-26T04:56:51.620639+0000","last_peered":"2024-04-26T04:56:51.620639+0000","last_clean":"2024-04-26T04:56:51.620639+0000","last_became_active":"2024-04-26T04:56:08.406919+0000","last_became_peered":"2024-04-26T04:56:08.406919+0000","last_unstale":"2024-04-26T04:56:51.620639+0000","last_undegraded":"2024-04-26T04:56:51.620639+0000","last_fullsized":"2024-04-26T04:56:51.620639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:19:56.376964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617000+0000","last_change":"2024-04-26T04:54:55.372743+0000","last_active":"2024-04-26T04:56:51.617000+0000","last_peered":"2024-04-26T04:56:51.617000+0000","last_clean":"2024-04-26T04:56:51.617000+0000","last_became_active":"2024-04-26T04:54:55.372440+0000","last_became_peered":"2024-04-26T04:54:55.372440+0000","last_unstale":"2024-04-26T04:56:51.617000+0000","last_undegraded":"2024-04-26T04:56:51.617000+0000","last_fullsized":"2024-04-26T04:56:51.617000+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:02:35.773050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:56:52.907901+0000","last_change":"2024-04-26T04:56:52.907901+0000","last_active":"2024-04-26T04:56:52.907901+0000","last_peered":"2024-04-26T04:56:52.907901+0000","last_clean":"2024-04-26T04:56:52.907901+0000","last_became_active":"2024-04-26T04:56:52.907625+0000","last_became_peered":"2024-04-26T04:56:52.907625+0000","last_unstale":"2024-04-26T04:56:52.907901+0000","last_undegraded":"2024-04-26T04:56:52.907901+0000","last_fullsized":"2024-04-26T04:56:52.907901+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:52:33.199158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617361+0000","last_change":"2024-04-26T04:56:08.409276+0000","last_active":"2024-04-26T04:56:51.617361+0000","last_peered":"2024-04-26T04:56:51.617361+0000","last_clean":"2024-04-26T04:56:51.617361+0000","last_became_active":"2024-04-26T04:56:08.409013+0000","last_became_peered":"2024-04-26T04:56:08.409013+0000","last_unstale":"2024-04-26T04:56:51.617361+0000","last_undegraded":"2024-04-26T04:56:51.617361+0000","last_fullsized":"2024-04-26T04:56:51.617361+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:36:59.616057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620479+0000","last_change":"2024-04-26T04:54:55.960319+0000","last_active":"2024-04-26T04:56:51.620479+0000","last_peered":"2024-04-26T04:56:51.620479+0000","last_clean":"2024-04-26T04:56:51.620479+0000","last_became_active":"2024-04-26T04:54:55.960025+0000","last_became_peered":"2024-04-26T04:54:55.960025+0000","last_unstale":"2024-04-26T04:56:51.620479+0000","last_undegraded":"2024-04-26T04:56:51.620479+0000","last_fullsized":"2024-04-26T04:56:51.620479+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:23:09.651998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620620+0000","last_change":"2024-04-26T04:56:09.703640+0000","last_active":"2024-04-26T04:56:51.620620+0000","last_peered":"2024-04-26T04:56:51.620620+0000","last_clean":"2024-04-26T04:56:51.620620+0000","last_became_active":"2024-04-26T04:56:09.703403+0000","last_became_peered":"2024-04-26T04:56:09.703403+0000","last_unstale":"2024-04-26T04:56:51.620620+0000","last_undegraded":"2024-04-26T04:56:51.620620+0000","last_fullsized":"2024-04-26T04:56:51.620620+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:46.948653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617504+0000","last_change":"2024-04-26T04:56:08.414956+0000","last_active":"2024-04-26T04:56:51.617504+0000","last_peered":"2024-04-26T04:56:51.617504+0000","last_clean":"2024-04-26T04:56:51.617504+0000","last_became_active":"2024-04-26T04:56:08.414690+0000","last_became_peered":"2024-04-26T04:56:08.414690+0000","last_unstale":"2024-04-26T04:56:51.617504+0000","last_undegraded":"2024-04-26T04:56:51.617504+0000","last_fullsized":"2024-04-26T04:56:51.617504+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:46:09.618427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621886+0000","last_change":"2024-04-26T04:56:08.422859+0000","last_active":"2024-04-26T04:56:51.621886+0000","last_peered":"2024-04-26T04:56:51.621886+0000","last_clean":"2024-04-26T04:56:51.621886+0000","last_became_active":"2024-04-26T04:56:08.422458+0000","last_became_peered":"2024-04-26T04:56:08.422458+0000","last_unstale":"2024-04-26T04:56:51.621886+0000","last_undegraded":"2024-04-26T04:56:51.621886+0000","last_fullsized":"2024-04-26T04:56:51.621886+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:14:22.398355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621200+0000","last_change":"2024-04-26T04:56:08.404428+0000","last_active":"2024-04-26T04:56:51.621200+0000","last_peered":"2024-04-26T04:56:51.621200+0000","last_clean":"2024-04-26T04:56:51.621200+0000","last_became_active":"2024-04-26T04:56:08.403880+0000","last_became_peered":"2024-04-26T04:56:08.403880+0000","last_unstale":"2024-04-26T04:56:51.621200+0000","last_undegraded":"2024-04-26T04:56:51.621200+0000","last_fullsized":"2024-04-26T04:56:51.621200+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:29:48.464556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621709+0000","last_change":"2024-04-26T04:56:08.422225+0000","last_active":"2024-04-26T04:56:51.621709+0000","last_peered":"2024-04-26T04:56:51.621709+0000","last_clean":"2024-04-26T04:56:51.621709+0000","last_became_active":"2024-04-26T04:56:08.421922+0000","last_became_peered":"2024-04-26T04:56:08.421922+0000","last_unstale":"2024-04-26T04:56:51.621709+0000","last_undegraded":"2024-04-26T04:56:51.621709+0000","last_fullsized":"2024-04-26T04:56:51.621709+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:41:25.502938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620610+0000","last_change":"2024-04-26T04:56:08.410384+0000","last_active":"2024-04-26T04:56:51.620610+0000","last_peered":"2024-04-26T04:56:51.620610+0000","last_clean":"2024-04-26T04:56:51.620610+0000","last_became_active":"2024-04-26T04:56:08.410076+0000","last_became_peered":"2024-04-26T04:56:08.410076+0000","last_unstale":"2024-04-26T04:56:51.620610+0000","last_undegraded":"2024-04-26T04:56:51.620610+0000","last_fullsized":"2024-04-26T04:56:51.620610+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:18:13.554446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621344+0000","last_change":"2024-04-26T04:56:08.736931+0000","last_active":"2024-04-26T04:56:51.621344+0000","last_peered":"2024-04-26T04:56:51.621344+0000","last_clean":"2024-04-26T04:56:51.621344+0000","last_became_active":"2024-04-26T04:56:08.736654+0000","last_became_peered":"2024-04-26T04:56:08.736654+0000","last_unstale":"2024-04-26T04:56:51.621344+0000","last_undegraded":"2024-04-26T04:56:51.621344+0000","last_fullsized":"2024-04-26T04:56:51.621344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:48.772903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617892+0000","last_change":"2024-04-26T04:56:08.414099+0000","last_active":"2024-04-26T04:56:51.617892+0000","last_peered":"2024-04-26T04:56:51.617892+0000","last_clean":"2024-04-26T04:56:51.617892+0000","last_became_active":"2024-04-26T04:56:08.413690+0000","last_became_peered":"2024-04-26T04:56:08.413690+0000","last_unstale":"2024-04-26T04:56:51.617892+0000","last_undegraded":"2024-04-26T04:56:51.617892+0000","last_fullsized":"2024-04-26T04:56:51.617892+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:39:48.905754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776681+0000","last_change":"2024-04-26T04:56:09.703515+0000","last_active":"2024-04-26T04:56:51.776681+0000","last_peered":"2024-04-26T04:56:51.776681+0000","last_clean":"2024-04-26T04:56:51.776681+0000","last_became_active":"2024-04-26T04:56:09.703229+0000","last_became_peered":"2024-04-26T04:56:09.703229+0000","last_unstale":"2024-04-26T04:56:51.776681+0000","last_undegraded":"2024-04-26T04:56:51.776681+0000","last_fullsized":"2024-04-26T04:56:51.776681+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:03:52.339898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617213+0000","last_change":"2024-04-26T04:56:08.408599+0000","last_active":"2024-04-26T04:56:51.617213+0000","last_peered":"2024-04-26T04:56:51.617213+0000","last_clean":"2024-04-26T04:56:51.617213+0000","last_became_active":"2024-04-26T04:56:08.408360+0000","last_became_peered":"2024-04-26T04:56:08.408360+0000","last_unstale":"2024-04-26T04:56:51.617213+0000","last_undegraded":"2024-04-26T04:56:51.617213+0000","last_fullsized":"2024-04-26T04:56:51.617213+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:01:42.788764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620407+0000","last_change":"2024-04-26T04:56:08.735526+0000","last_active":"2024-04-26T04:56:51.620407+0000","last_peered":"2024-04-26T04:56:51.620407+0000","last_clean":"2024-04-26T04:56:51.620407+0000","last_became_active":"2024-04-26T04:56:08.735232+0000","last_became_peered":"2024-04-26T04:56:08.735232+0000","last_unstale":"2024-04-26T04:56:51.620407+0000","last_undegraded":"2024-04-26T04:56:51.620407+0000","last_fullsized":"2024-04-26T04:56:51.620407+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:37:25.532847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777056+0000","last_change":"2024-04-26T04:56:08.400274+0000","last_active":"2024-04-26T04:56:51.777056+0000","last_peered":"2024-04-26T04:56:51.777056+0000","last_clean":"2024-04-26T04:56:51.777056+0000","last_became_active":"2024-04-26T04:56:08.400012+0000","last_became_peered":"2024-04-26T04:56:08.400012+0000","last_unstale":"2024-04-26T04:56:51.777056+0000","last_undegraded":"2024-04-26T04:56:51.777056+0000","last_fullsized":"2024-04-26T04:56:51.777056+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:06:01.413415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621027+0000","last_change":"2024-04-26T04:56:08.736920+0000","last_active":"2024-04-26T04:56:51.621027+0000","last_peered":"2024-04-26T04:56:51.621027+0000","last_clean":"2024-04-26T04:56:51.621027+0000","last_became_active":"2024-04-26T04:56:08.736599+0000","last_became_peered":"2024-04-26T04:56:08.736599+0000","last_unstale":"2024-04-26T04:56:51.621027+0000","last_undegraded":"2024-04-26T04:56:51.621027+0000","last_fullsized":"2024-04-26T04:56:51.621027+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:38:39.874495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777098+0000","last_change":"2024-04-26T04:56:08.407242+0000","last_active":"2024-04-26T04:56:51.777098+0000","last_peered":"2024-04-26T04:56:51.777098+0000","last_clean":"2024-04-26T04:56:51.777098+0000","last_became_active":"2024-04-26T04:56:08.406253+0000","last_became_peered":"2024-04-26T04:56:08.406253+0000","last_unstale":"2024-04-26T04:56:51.777098+0000","last_undegraded":"2024-04-26T04:56:51.777098+0000","last_fullsized":"2024-04-26T04:56:51.777098+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:10:24.314178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617247+0000","last_change":"2024-04-26T04:56:08.735825+0000","last_active":"2024-04-26T04:56:51.617247+0000","last_peered":"2024-04-26T04:56:51.617247+0000","last_clean":"2024-04-26T04:56:51.617247+0000","last_became_active":"2024-04-26T04:56:08.735226+0000","last_became_peered":"2024-04-26T04:56:08.735226+0000","last_unstale":"2024-04-26T04:56:51.617247+0000","last_undegraded":"2024-04-26T04:56:51.617247+0000","last_fullsized":"2024-04-26T04:56:51.617247+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:17:43.402733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777135+0000","last_change":"2024-04-26T04:56:08.407027+0000","last_active":"2024-04-26T04:56:51.777135+0000","last_peered":"2024-04-26T04:56:51.777135+0000","last_clean":"2024-04-26T04:56:51.777135+0000","last_became_active":"2024-04-26T04:56:08.406743+0000","last_became_peered":"2024-04-26T04:56:08.406743+0000","last_unstale":"2024-04-26T04:56:51.777135+0000","last_undegraded":"2024-04-26T04:56:51.777135+0000","last_fullsized":"2024-04-26T04:56:51.777135+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:59:50.242788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.618302+0000","last_change":"2024-04-26T04:56:09.703080+0000","last_active":"2024-04-26T04:56:51.618302+0000","last_peered":"2024-04-26T04:56:51.618302+0000","last_clean":"2024-04-26T04:56:51.618302+0000","last_became_active":"2024-04-26T04:56:09.702772+0000","last_became_peered":"2024-04-26T04:56:09.702772+0000","last_unstale":"2024-04-26T04:56:51.618302+0000","last_undegraded":"2024-04-26T04:56:51.618302+0000","last_fullsized":"2024-04-26T04:56:51.618302+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:09:10.245555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621591+0000","last_change":"2024-04-26T04:56:08.405481+0000","last_active":"2024-04-26T04:56:51.621591+0000","last_peered":"2024-04-26T04:56:51.621591+0000","last_clean":"2024-04-26T04:56:51.621591+0000","last_became_active":"2024-04-26T04:56:08.405217+0000","last_became_peered":"2024-04-26T04:56:08.405217+0000","last_unstale":"2024-04-26T04:56:51.621591+0000","last_undegraded":"2024-04-26T04:56:51.621591+0000","last_fullsized":"2024-04-26T04:56:51.621591+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:37:59.899823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776809+0000","last_change":"2024-04-26T04:56:09.702688+0000","last_active":"2024-04-26T04:56:51.776809+0000","last_peered":"2024-04-26T04:56:51.776809+0000","last_clean":"2024-04-26T04:56:51.776809+0000","last_became_active":"2024-04-26T04:56:09.702369+0000","last_became_peered":"2024-04-26T04:56:09.702369+0000","last_unstale":"2024-04-26T04:56:51.776809+0000","last_undegraded":"2024-04-26T04:56:51.776809+0000","last_fullsized":"2024-04-26T04:56:51.776809+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:46:02.167155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776639+0000","last_change":"2024-04-26T04:56:08.400160+0000","last_active":"2024-04-26T04:56:51.776639+0000","last_peered":"2024-04-26T04:56:51.776639+0000","last_clean":"2024-04-26T04:56:51.776639+0000","last_became_active":"2024-04-26T04:56:08.399849+0000","last_became_peered":"2024-04-26T04:56:08.399849+0000","last_unstale":"2024-04-26T04:56:51.776639+0000","last_undegraded":"2024-04-26T04:56:51.776639+0000","last_fullsized":"2024-04-26T04:56:51.776639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:26:13.567428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617178+0000","last_change":"2024-04-26T04:56:08.735832+0000","last_active":"2024-04-26T04:56:51.617178+0000","last_peered":"2024-04-26T04:56:51.617178+0000","last_clean":"2024-04-26T04:56:51.617178+0000","last_became_active":"2024-04-26T04:56:08.735579+0000","last_became_peered":"2024-04-26T04:56:08.735579+0000","last_unstale":"2024-04-26T04:56:51.617178+0000","last_undegraded":"2024-04-26T04:56:51.617178+0000","last_fullsized":"2024-04-26T04:56:51.617178+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:10:51.768335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776678+0000","last_change":"2024-04-26T04:56:08.418624+0000","last_active":"2024-04-26T04:56:51.776678+0000","last_peered":"2024-04-26T04:56:51.776678+0000","last_clean":"2024-04-26T04:56:51.776678+0000","last_became_active":"2024-04-26T04:56:08.418366+0000","last_became_peered":"2024-04-26T04:56:08.418366+0000","last_unstale":"2024-04-26T04:56:51.776678+0000","last_undegraded":"2024-04-26T04:56:51.776678+0000","last_fullsized":"2024-04-26T04:56:51.776678+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:34:57.765643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617949+0000","last_change":"2024-04-26T04:56:09.701457+0000","last_active":"2024-04-26T04:56:51.617949+0000","last_peered":"2024-04-26T04:56:51.617949+0000","last_clean":"2024-04-26T04:56:51.617949+0000","last_became_active":"2024-04-26T04:56:09.701132+0000","last_became_peered":"2024-04-26T04:56:09.701132+0000","last_unstale":"2024-04-26T04:56:51.617949+0000","last_undegraded":"2024-04-26T04:56:51.617949+0000","last_fullsized":"2024-04-26T04:56:51.617949+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:30:40.646109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776865+0000","last_change":"2024-04-26T04:56:08.405604+0000","last_active":"2024-04-26T04:56:51.776865+0000","last_peered":"2024-04-26T04:56:51.776865+0000","last_clean":"2024-04-26T04:56:51.776865+0000","last_became_active":"2024-04-26T04:56:08.405321+0000","last_became_peered":"2024-04-26T04:56:08.405321+0000","last_unstale":"2024-04-26T04:56:51.776865+0000","last_undegraded":"2024-04-26T04:56:51.776865+0000","last_fullsized":"2024-04-26T04:56:51.776865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:35:18.908328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620732+0000","last_change":"2024-04-26T04:56:08.421752+0000","last_active":"2024-04-26T04:56:51.620732+0000","last_peered":"2024-04-26T04:56:51.620732+0000","last_clean":"2024-04-26T04:56:51.620732+0000","last_became_active":"2024-04-26T04:56:08.421495+0000","last_became_peered":"2024-04-26T04:56:08.421495+0000","last_unstale":"2024-04-26T04:56:51.620732+0000","last_undegraded":"2024-04-26T04:56:51.620732+0000","last_fullsized":"2024-04-26T04:56:51.620732+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:31:54.194701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621445+0000","last_change":"2024-04-26T04:56:08.399546+0000","last_active":"2024-04-26T04:56:51.621445+0000","last_peered":"2024-04-26T04:56:51.621445+0000","last_clean":"2024-04-26T04:56:51.621445+0000","last_became_active":"2024-04-26T04:56:08.399266+0000","last_became_peered":"2024-04-26T04:56:08.399266+0000","last_unstale":"2024-04-26T04:56:51.621445+0000","last_undegraded":"2024-04-26T04:56:51.621445+0000","last_fullsized":"2024-04-26T04:56:51.621445+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:46:52.731455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620285+0000","last_change":"2024-04-26T04:56:09.702277+0000","last_active":"2024-04-26T04:56:51.620285+0000","last_peered":"2024-04-26T04:56:51.620285+0000","last_clean":"2024-04-26T04:56:51.620285+0000","last_became_active":"2024-04-26T04:56:09.701935+0000","last_became_peered":"2024-04-26T04:56:09.701935+0000","last_unstale":"2024-04-26T04:56:51.620285+0000","last_undegraded":"2024-04-26T04:56:51.620285+0000","last_fullsized":"2024-04-26T04:56:51.620285+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:55:40.135676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776949+0000","last_change":"2024-04-26T04:56:08.407185+0000","last_active":"2024-04-26T04:56:51.776949+0000","last_peered":"2024-04-26T04:56:51.776949+0000","last_clean":"2024-04-26T04:56:51.776949+0000","last_became_active":"2024-04-26T04:56:08.406921+0000","last_became_peered":"2024-04-26T04:56:08.406921+0000","last_unstale":"2024-04-26T04:56:51.776949+0000","last_undegraded":"2024-04-26T04:56:51.776949+0000","last_fullsized":"2024-04-26T04:56:51.776949+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:19:22.160857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617968+0000","last_change":"2024-04-26T04:56:09.702829+0000","last_active":"2024-04-26T04:56:51.617968+0000","last_peered":"2024-04-26T04:56:51.617968+0000","last_clean":"2024-04-26T04:56:51.617968+0000","last_became_active":"2024-04-26T04:56:09.702562+0000","last_became_peered":"2024-04-26T04:56:09.702562+0000","last_unstale":"2024-04-26T04:56:51.617968+0000","last_undegraded":"2024-04-26T04:56:51.617968+0000","last_fullsized":"2024-04-26T04:56:51.617968+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:00:30.433073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617436+0000","last_change":"2024-04-26T04:56:08.405882+0000","last_active":"2024-04-26T04:56:51.617436+0000","last_peered":"2024-04-26T04:56:51.617436+0000","last_clean":"2024-04-26T04:56:51.617436+0000","last_became_active":"2024-04-26T04:56:08.404791+0000","last_became_peered":"2024-04-26T04:56:08.404791+0000","last_unstale":"2024-04-26T04:56:51.617436+0000","last_undegraded":"2024-04-26T04:56:51.617436+0000","last_fullsized":"2024-04-26T04:56:51.617436+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:00:32.949944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:56:52.908129+0000","last_change":"2024-04-26T04:56:52.908129+0000","last_active":"2024-04-26T04:56:52.908129+0000","last_peered":"2024-04-26T04:56:52.908129+0000","last_clean":"2024-04-26T04:56:52.908129+0000","last_became_active":"2024-04-26T04:56:52.907768+0000","last_became_peered":"2024-04-26T04:56:52.907768+0000","last_unstale":"2024-04-26T04:56:52.908129+0000","last_undegraded":"2024-04-26T04:56:52.908129+0000","last_fullsized":"2024-04-26T04:56:52.908129+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:37:51.277368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.777008+0000","last_change":"2024-04-26T04:56:08.417932+0000","last_active":"2024-04-26T04:56:51.777008+0000","last_peered":"2024-04-26T04:56:51.777008+0000","last_clean":"2024-04-26T04:56:51.777008+0000","last_became_active":"2024-04-26T04:56:08.417649+0000","last_became_peered":"2024-04-26T04:56:08.417649+0000","last_unstale":"2024-04-26T04:56:51.777008+0000","last_undegraded":"2024-04-26T04:56:51.777008+0000","last_fullsized":"2024-04-26T04:56:51.777008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:46:27.177515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620422+0000","last_change":"2024-04-26T04:56:08.423360+0000","last_active":"2024-04-26T04:56:51.620422+0000","last_peered":"2024-04-26T04:56:51.620422+0000","last_clean":"2024-04-26T04:56:51.620422+0000","last_became_active":"2024-04-26T04:56:08.423072+0000","last_became_peered":"2024-04-26T04:56:08.423072+0000","last_unstale":"2024-04-26T04:56:51.620422+0000","last_undegraded":"2024-04-26T04:56:51.620422+0000","last_fullsized":"2024-04-26T04:56:51.620422+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:18:04.178846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.617791+0000","last_change":"2024-04-26T04:56:08.417723+0000","last_active":"2024-04-26T04:56:51.617791+0000","last_peered":"2024-04-26T04:56:51.617791+0000","last_clean":"2024-04-26T04:56:51.617791+0000","last_became_active":"2024-04-26T04:56:08.416629+0000","last_became_peered":"2024-04-26T04:56:08.416629+0000","last_unstale":"2024-04-26T04:56:51.617791+0000","last_undegraded":"2024-04-26T04:56:51.617791+0000","last_fullsized":"2024-04-26T04:56:51.617791+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:52:49.448467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.776837+0000","last_change":"2024-04-26T04:56:09.702682+0000","last_active":"2024-04-26T04:56:51.776837+0000","last_peered":"2024-04-26T04:56:51.776837+0000","last_clean":"2024-04-26T04:56:51.776837+0000","last_became_active":"2024-04-26T04:56:09.702380+0000","last_became_peered":"2024-04-26T04:56:09.702380+0000","last_unstale":"2024-04-26T04:56:51.776837+0000","last_undegraded":"2024-04-26T04:56:51.776837+0000","last_fullsized":"2024-04-26T04:56:51.776837+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:56:31.820640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.620330+0000","last_change":"2024-04-26T04:56:08.409816+0000","last_active":"2024-04-26T04:56:51.620330+0000","last_peered":"2024-04-26T04:56:51.620330+0000","last_clean":"2024-04-26T04:56:51.620330+0000","last_became_active":"2024-04-26T04:56:08.409544+0000","last_became_peered":"2024-04-26T04:56:08.409544+0000","last_unstale":"2024-04-26T04:56:51.620330+0000","last_undegraded":"2024-04-26T04:56:51.620330+0000","last_fullsized":"2024-04-26T04:56:51.620330+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:51:16.478567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:56:52.857921+0000","last_change":"2024-04-26T04:56:52.857921+0000","last_active":"2024-04-26T04:56:52.857921+0000","last_peered":"2024-04-26T04:56:52.857921+0000","last_clean":"2024-04-26T04:56:52.857921+0000","last_became_active":"2024-04-26T04:56:52.857664+0000","last_became_peered":"2024-04-26T04:56:52.857664+0000","last_unstale":"2024-04-26T04:56:52.857921+0000","last_undegraded":"2024-04-26T04:56:52.857921+0000","last_fullsized":"2024-04-26T04:56:52.857921+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:11:11.311316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:56:51.621751+0000","last_change":"2024-04-26T04:56:08.405077+0000","last_active":"2024-04-26T04:56:51.621751+0000","last_peered":"2024-04-26T04:56:51.621751+0000","last_clean":"2024-04-26T04:56:51.621751+0000","last_became_active":"2024-04-26T04:56:08.404805+0000","last_became_peered":"2024-04-26T04:56:08.404805+0000","last_unstale":"2024-04-26T04:56:51.621751+0000","last_undegraded":"2024-04-26T04:56:51.621751+0000","last_fullsized":"2024-04-26T04:56:51.621751+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:16:41.050046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.504,"5min":0.499,"15min":0.498},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.68500000000000005}]},{"osd":1,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51500000000000001,"15min":0.51300000000000001},"min":{"1min":0.318,"5min":0.316,"15min":0.316},"max":{"1min":0.77500000000000002,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.58599999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.80400000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.70199999999999996}]},{"osd":3,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53100000000000003,"15min":0.53000000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56399999999999995,"15min":0.56299999999999994},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.57699999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.48399999999999999,"15min":0.47599999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.79200000000000004,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.45900000000000002,"15min":0.44900000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73099999999999998,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.496}]},{"osd":5,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.47099999999999997,"15min":0.46000000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.84399999999999997,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.50900000000000001,"15min":0.501},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.753,"5min":0.878,"15min":0.878},"last":0.55400000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.503,"15min":0.495},"min":{"1min":0.35899999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.85899999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.55300000000000005,"15min":0.54400000000000004},"min":{"1min":0.38600000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.82999999999999996,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.622}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1300,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1331200,"data_stored":908128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.41799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69599999999999995,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51600000000000001,"15min":0.503},"min":{"1min":0.438,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.64400000000000002,"5min":0.745,"15min":0.745},"last":0.53700000000000003}]},{"osd":1,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.43099999999999999,"5min":0.38,"15min":0.38},"max":{"1min":0.755,"5min":1.008,"15min":1.008},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55100000000000005,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72399999999999998,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.57899999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.39800000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.625,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54500000000000004,"15min":0.55000000000000004},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77900000000000003,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.55200000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53000000000000003,"15min":0.52700000000000002},"min":{"1min":0.40100000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.66100000000000003,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38400000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.63800000000000001,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.56499999999999995}]},{"osd":4,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44500000000000001,"15min":0.439},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.73099999999999998,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.45300000000000001,"15min":0.442},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66800000000000004,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.63400000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:57:03 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68700000000000006,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.48699999999999999,"15min":0.47999999999999998},"min":{"1min":0.31900000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.70499999999999996,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.59499999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.48499999999999999,"15min":0.47699999999999998},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.65300000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":315931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:57:02 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.35199999999999998,"15min":0.33300000000000002},"min":{"1min":0.222,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.52100000000000002,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.36799999999999999,"15min":0.33400000000000002},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.495,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.34999999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.27800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.56799999999999995,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.53900000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.436}]},{"osd":3,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42399999999999999,"15min":0.436},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.499,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.41299999999999998,"15min":0.42899999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.44700000000000001,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.44700000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:56:09 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.41299999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.44600000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.67300000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:56:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.37,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58099999999999996,"15min":0.58399999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.64800000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:57:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.40000000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.621},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82399999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.69499999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57399999999999995,"15min":0.56599999999999995},"min":{"1min":0.42099999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.57299999999999995,"15min":0.56299999999999994},"min":{"1min":0.42299999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.61099999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":907421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.38200000000000001,"15min":0.34200000000000003},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.62,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.374,"15min":0.33000000000000002},"min":{"1min":0.251,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.38600000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.623,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.41699999999999998,"15min":0.40899999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.56499999999999995,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40600000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.44500000000000001,"15min":0.46899999999999997},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56999999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.41499999999999998,"15min":0.433},"min":{"1min":0.26600000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55800000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.36599999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.622,"15min":0.63900000000000001},"min":{"1min":0.41999999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.623,"15min":0.63700000000000001},"min":{"1min":0.42999999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.621}]},{"osd":5,"last update":"Fri Apr 26 04:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59399999999999997,"15min":0.59599999999999997},"min":{"1min":0.45000000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.84199999999999997,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.40100000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76400000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.58799999999999997,"15min":0.58899999999999997},"min":{"1min":0.41999999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.76100000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.44900000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.60399999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.55400000000000005,"15min":0.54700000000000004},"min":{"1min":0.45900000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.58299999999999996,"15min":0.57499999999999996},"min":{"1min":0.42799999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.64100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.34499999999999997,"15min":0.33800000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51800000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.379},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.36199999999999999,"15min":0.35999999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.54200000000000004,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.52300000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.35799999999999998,"15min":0.35699999999999998},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.40400000000000003,"15min":0.39600000000000002},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56200000000000006,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.441}]},{"osd":3,"last update":"Fri Apr 26 04:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45900000000000002,"15min":0.46700000000000003},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.44,"15min":0.45200000000000001},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.496}]},{"osd":4,"last update":"Fri Apr 26 04:56:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.56799999999999995,"15min":0.56000000000000005},"min":{"1min":0.44,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.74099999999999999,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.754,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.64600000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:56:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54900000000000004,"15min":0.54100000000000004},"min":{"1min":0.39400000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57799999999999996,"15min":0.58299999999999996},"min":{"1min":0.44400000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.60799999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:57:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60299999999999998,"15min":0.61099999999999999},"min":{"1min":0.42999999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81200000000000006,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.59999999999999998,"15min":0.61399999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84599999999999997,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.59799999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76800000000000002,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.63100000000000001,"15min":0.63600000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.81599999999999995,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.63300000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.38100000000000001,"15min":0.39400000000000002},"min":{"1min":0.253,"5min":0.245,"15min":0.245},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41699999999999998,"15min":0.43099999999999999},"min":{"1min":0.308,"5min":0.245,"15min":0.245},"max":{"1min":0.54700000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.33800000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.40600000000000003,"15min":0.40799999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.53700000000000003,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.40699999999999997,"15min":0.42199999999999999},"min":{"1min":0.27900000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.57199999999999995,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.32200000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.441,"15min":0.45400000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.66100000000000003,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42999999999999999,"15min":0.45200000000000001},"min":{"1min":0.28699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.35199999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:56:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.70599999999999996}]},{"osd":5,"last update":"Fri Apr 26 04:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.57899999999999996,"15min":0.59499999999999997},"min":{"1min":0.38500000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.77000000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.873},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.40300000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.60499999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:57:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58999999999999997,"15min":0.59499999999999997},"min":{"1min":0.44800000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.77700000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:56:22 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58399999999999996,"15min":0.57999999999999996},"min":{"1min":0.46100000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.754},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61599999999999999,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.82599999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":745472,"data_stored":317894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.38200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69999999999999996,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.53800000000000003},"min":{"1min":0.41299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.70199999999999996,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.66600000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.55900000000000005},"min":{"1min":0.36499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61799999999999999,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55600000000000005,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.71699999999999997,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.68600000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.68400000000000005,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.44500000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.68200000000000005,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.61499999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.56699999999999995,"15min":0.55300000000000005},"min":{"1min":0.44,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73599999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59299999999999997,"15min":0.60099999999999998},"min":{"1min":0.44500000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.754,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.69799999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:56:39 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.40999999999999998,"15min":0.39400000000000002},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.56399999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.42999999999999999,"15min":0.40999999999999998},"min":{"1min":0.32600000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.61399999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.34699999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:57:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47699999999999998,"15min":0.47499999999999998},"min":{"1min":0.34200000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66800000000000004,"5min":0.755,"15min":0.755},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.47699999999999998,"15min":0.46999999999999997},"min":{"1min":0.34499999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.69599999999999995,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.46500000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:56:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.40699999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.443},{"interface":"front","average":{"1min":0.501,"5min":0.51600000000000001,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.48599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.29199999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.68100000000000005,"5min":0.748,"15min":0.748},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.66600000000000004,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.443}]},{"osd":1,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53900000000000003,"15min":0.53500000000000003},"min":{"1min":0.36399999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67700000000000005,"5min":0.753,"15min":0.753},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.503,"15min":0.503},"min":{"1min":0.38300000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.65300000000000002,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.40999999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55100000000000005,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71099999999999997,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70199999999999996,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.65600000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67300000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56299999999999994,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.68500000000000005,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.495}]},{"osd":4,"last update":"Fri Apr 26 04:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.40200000000000002,"15min":0.39000000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.58599999999999997,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.42899999999999999,"15min":0.40699999999999997},"min":{"1min":0.32500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64200000000000002,"5min":0.75,"15min":0.75},"last":0.54200000000000004}]},{"osd":6,"last update":"Fri Apr 26 04:56:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.45500000000000002,"15min":0.42599999999999999},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.46300000000000002,"15min":0.438},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:56:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.38700000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.76500000000000001,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.624},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.34100000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.63900000000000001,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52600000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:57:13.476 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:57:13.476 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:57:13.476 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:57:13.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:13.732 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:13.854 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:13 smithi152 ceph-mon[27560]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:57:13.855 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:13 smithi152 ceph-mon[39324]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:57:14.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:13 smithi195 ceph-mon[31427]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:57:14.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:14 smithi152 ceph-mon[27560]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:14 smithi152 ceph-mon[39324]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:15.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:14 smithi195 ceph-mon[31427]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:15.823 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T04:57:15.823 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:57:16.580 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:57:16.580 INFO:teuthology.run_tasks:Running task print... 2024-04-26T04:57:16.591 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-26T04:57:16.591 INFO:teuthology.run_tasks:Running task parallel... 2024-04-26T04:57:16.605 INFO:teuthology.task.parallel:starting parallel... 2024-04-26T04:57:16.606 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:57:16.606 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-26T04:57:16.610 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-04-26T04:57:16.610 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-26T04:57:16.611 DEBUG:teuthology.orchestra.run.smithi152:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:57:16.611 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:57:16.611 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-26T04:57:16.611 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-26T04:57:16.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-26T04:57:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:57:16.627 INFO:teuthology.orchestra.run.smithi152.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-04-26T04:57:16.627 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:57:16.682 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-04-26T04:57:16.682 DEBUG:teuthology.orchestra.run.smithi152:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-04-26T04:57:16.737 INFO:tasks.workunit:timeout=3h 2024-04-26T04:57:16.737 INFO:tasks.workunit:cleanup=True 2024-04-26T04:57:16.738 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-04-26T04:57:16.794 INFO:tasks.workunit.client.0.smithi152.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-26T04:57:16.870 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:16.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:16 smithi152 ceph-mon[27560]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:16.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:16 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1563689940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:16.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:16 smithi152 ceph-mon[39324]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:16.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:16 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1563689940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:17.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:16 smithi195 ceph-mon[31427]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:17.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:16 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1563689940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:18.943 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-mon[27560]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:18.944 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-mon[39324]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:18.944 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:57:18] ENGINE Bus STOPPING 2024-04-26T04:57:18.944 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:57:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:57:18.944 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:57:18] ENGINE Bus STOPPED 2024-04-26T04:57:18.944 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:57:18] ENGINE Bus STARTING 2024-04-26T04:57:18.944 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:57:18] ENGINE Serving on http://:::9283 2024-04-26T04:57:18.944 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:18 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:57:18] ENGINE Bus STARTED 2024-04-26T04:57:19.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:18 smithi195 ceph-mon[31427]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:19.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/993630734' entity='client.admin' 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[27560]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/993630734' entity='client.admin' 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:19.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:19.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:19.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:19.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:19 smithi152 ceph-mon[39324]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:19.917 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:20.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:19 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/993630734' entity='client.admin' 2024-04-26T04:57:20.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:19 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:20.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:19 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:20.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:19 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:20.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:19 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:20.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:19 smithi195 ceph-mon[31427]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:21.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:57:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:21 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:21.842 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:21 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:21.843 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:21 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:22.571 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:22 smithi152 ceph-mon[27560]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:22.571 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:22 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:22] "GET /metrics HTTP/1.1" 200 239106 "" "Prometheus/2.43.0" 2024-04-26T04:57:22.571 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:22 smithi152 ceph-mon[39324]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:22.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T04:57:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:22 smithi195 ceph-mon[31427]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:22.849 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:24.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:24 smithi152 ceph-mon[39324]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:24.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:24 smithi152 ceph-mon[27560]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:24.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:24 smithi195 ceph-mon[31427]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:24.816 INFO:teuthology.orchestra.run.smithi152.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:57:25.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:57:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[27560]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:57:25.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:25 smithi152 ceph-mon[39324]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:25.914 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:57:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:57:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:25 smithi195 ceph-mon[31427]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:28.048 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.a smithi152 *:9093,9094 running (84s) 42s ago 2m 16.4M - 0.25.0 c8568f914cd2 92be36185dbe 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.a smithi195 *:3000 running (2m) 57s ago 2m 82.2M - 9.4.7 954c08fa6188 8dd84a128929 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi152.rnhpxy smithi152 running (70s) 42s ago 70s 13.2M - 17.2.7-902-gc61472c8 0de407c1b3af 6f0b0b86b7ef 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi195.bqeurw smithi195 running (68s) 57s ago 67s 13.3M - 17.2.7-902-gc61472c8 0de407c1b3af db58137dcdb3 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.x smithi195 *:8443 running (6m) 57s ago 6m 431M - 17.2.7-902-gc61472c8 0de407c1b3af 0f1dfdd9436f 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.y smithi152 *:9283 running (8m) 42s ago 8m 474M - 17.2.7-902-gc61472c8 0de407c1b3af ee2b14613007 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:mon.a smithi152 running (8m) 42s ago 8m 42.4M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 343ee858bd2c 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:mon.b smithi195 running (6m) 57s ago 6m 40.6M 2048M 17.2.7-902-gc61472c8 0de407c1b3af bf477f6a2b3d 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:mon.c smithi152 running (6m) 42s ago 6m 40.1M 2048M 17.2.7-902-gc61472c8 0de407c1b3af f907cd5ba87f 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.a smithi152 *:9100 running (114s) 42s ago 112s 15.9M - 1.5.0 0da6a335fe13 84ac3154b2e4 2024-04-26T04:57:28.049 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.b smithi195 *:9100 running (109s) 57s ago 108s 7843k - 1.5.0 0da6a335fe13 b3a8846a9680 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (6m) 42s ago 6m 42.7M 4016M 17.2.7-902-gc61472c8 0de407c1b3af a6ef80972333 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (5m) 42s ago 5m 41.5M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 827c6f69d0b5 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (5m) 42s ago 5m 40.7M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 3b4441be6ba1 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi152 running (4m) 42s ago 4m 40.6M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 1b39c722ab3d 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi195 running (4m) 57s ago 4m 42.7M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 958de1ceb551 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi195 running (4m) 57s ago 4m 40.3M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 5a97e623bdc9 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.6 smithi195 running (3m) 57s ago 3m 40.3M 4016M 17.2.7-902-gc61472c8 0de407c1b3af f77aacbd5fd5 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:osd.7 smithi195 running (3m) 57s ago 3m 43.4M 4016M 17.2.7-902-gc61472c8 0de407c1b3af e1529e54096a 2024-04-26T04:57:28.050 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.a smithi195 *:9095 running (79s) 57s ago 2m 33.5M - 2.43.0 a07b618ecd1d a731aeae0e89 2024-04-26T04:57:28.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:28 smithi152 ceph-mon[27560]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:28.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:28 smithi152 ceph-mon[39324]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:28.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-26T04:57:28.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:28 smithi195 ceph-mon[31427]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:29.056 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:29.505 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:29 smithi152 ceph-mon[27560]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:29.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:29 smithi152 ceph-mon[39324]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:29.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:29 smithi195 ceph-mon[31427]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:30.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:30 smithi152 ceph-mon[27560]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:30.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:30 smithi152 ceph-mon[39324]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:30.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:30 smithi195 ceph-mon[31427]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:31.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:57:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:31.344 DEBUG:teuthology.orchestra.run.smithi152:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-26T04:57:31.434 INFO:tasks.workunit.client.0.smithi152.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-26T04:57:31.436 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:57:31.436 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-26T04:57:31.735 INFO:teuthology.orchestra.run.smithi152.stdout:wait for minority of mons to upgrade 2024-04-26T04:57:32.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:32 smithi152 ceph-mon[27560]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:32.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:32] "GET /metrics HTTP/1.1" 200 239106 "" "Prometheus/2.43.0" 2024-04-26T04:57:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:32 smithi152 ceph-mon[39324]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:32.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-26T04:57:32.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:32 smithi195 ceph-mon[31427]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:33.043 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-26T04:57:33.046 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:33.138 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-26T04:57:33.179 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:57:33.180 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:57:33.180 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-26T04:57:33.227 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:57:33.230 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-26T04:57:33.230 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-26T04:57:33.246 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-04-26T04:57:33.247 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-04-26T04:57:33.247 DEBUG:teuthology.orchestra.run.smithi152:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-04-26T04:57:33.305 INFO:tasks.workunit.client.0.smithi152.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-04-26T04:57:33.305 INFO:tasks.workunit.client.0.smithi152.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-04-26T04:57:33.305 INFO:tasks.workunit.client.0.smithi152.stderr:+ TMPDIR=/tmp/rbd_import_export_85090 2024-04-26T04:57:33.305 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm -rf /tmp/rbd_import_export_85090 2024-04-26T04:57:33.306 INFO:tasks.workunit.client.0.smithi152.stderr:+ mkdir /tmp/rbd_import_export_85090 2024-04-26T04:57:33.307 INFO:tasks.workunit.client.0.smithi152.stderr:+ trap 'rm -rf /tmp/rbd_import_export_85090' INT TERM EXIT 2024-04-26T04:57:33.307 INFO:tasks.workunit.client.0.smithi152.stderr:+ mkdir foo.85090 2024-04-26T04:57:33.307 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import foo.85090 foo.dir 2024-04-26T04:57:33.334 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: cannot import a directory 2024-04-26T04:57:33.335 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 0% complete...failed. 2024-04-26T04:57:33.335 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: import failed: (21) Is a directory 2024-04-26T04:57:33.356 INFO:tasks.workunit.client.0.smithi152.stderr:+ true 2024-04-26T04:57:33.356 INFO:tasks.workunit.client.0.smithi152.stderr:+ rmdir foo.85090 2024-04-26T04:57:33.373 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_85090/img bs=1k count=1 seek=10 2024-04-26T04:57:33.374 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records in 2024-04-26T04:57:33.374 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records out 2024-04-26T04:57:33.374 INFO:tasks.workunit.client.0.smithi152.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.9658e-05 s, 20.6 MB/s 2024-04-26T04:57:33.374 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_85090/img bs=1k count=10 seek=100 2024-04-26T04:57:33.375 INFO:tasks.workunit.client.0.smithi152.stderr:10+0 records in 2024-04-26T04:57:33.375 INFO:tasks.workunit.client.0.smithi152.stderr:10+0 records out 2024-04-26T04:57:33.375 INFO:tasks.workunit.client.0.smithi152.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.371e-05 s, 191 MB/s 2024-04-26T04:57:33.375 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_85090/img bs=1k count=100 seek=1000 2024-04-26T04:57:33.376 INFO:tasks.workunit.client.0.smithi152.stderr:60+1 records in 2024-04-26T04:57:33.376 INFO:tasks.workunit.client.0.smithi152.stderr:60+1 records out 2024-04-26T04:57:33.376 INFO:tasks.workunit.client.0.smithi152.stderr:61656 bytes (62 kB, 60 KiB) copied, 0.000147842 s, 417 MB/s 2024-04-26T04:57:33.376 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_85090/img bs=1k seek=10000 2024-04-26T04:57:33.377 INFO:tasks.workunit.client.0.smithi152.stderr:137+1 records in 2024-04-26T04:57:33.377 INFO:tasks.workunit.client.0.smithi152.stderr:137+1 records out 2024-04-26T04:57:33.377 INFO:tasks.workunit.client.0.smithi152.stderr:140952 bytes (141 kB, 138 KiB) copied, 0.0003538 s, 398 MB/s 2024-04-26T04:57:33.377 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_85090/img bs=1k seek=100000 2024-04-26T04:57:33.377 INFO:tasks.workunit.client.0.smithi152.stderr:60+1 records in 2024-04-26T04:57:33.377 INFO:tasks.workunit.client.0.smithi152.stderr:60+1 records out 2024-04-26T04:57:33.378 INFO:tasks.workunit.client.0.smithi152.stderr:61608 bytes (62 kB, 60 KiB) copied, 0.000148233 s, 416 MB/s 2024-04-26T04:57:33.378 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_85090/img bs=1k seek=1000000 2024-04-26T04:57:33.379 INFO:tasks.workunit.client.0.smithi152.stderr:154+1 records in 2024-04-26T04:57:33.379 INFO:tasks.workunit.client.0.smithi152.stderr:154+1 records out 2024-04-26T04:57:33.379 INFO:tasks.workunit.client.0.smithi152.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.000455926 s, 347 MB/s 2024-04-26T04:57:33.379 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm testimg 2024-04-26T04:57:33.428 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 0% complete...failed. 2024-04-26T04:57:33.428 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: delete error: (2) No such file or directory 2024-04-26T04:57:33.431 INFO:tasks.workunit.client.0.smithi152.stderr:+ true 2024-04-26T04:57:33.431 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format /tmp/rbd_import_export_85090/img testimg 2024-04-26T04:57:34.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:34 smithi152 ceph-mon[27560]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:34.644 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:34 smithi152 ceph-mon[39324]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:34.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:34 smithi195 ceph-mon[31427]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:35.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:35 smithi195 ceph-mon[31427]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:57:35.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:35 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2446557302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:35.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:35 smithi152 ceph-mon[27560]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:57:35.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:35 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2446557302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:35.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:35 smithi152 ceph-mon[39324]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:57:35.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:35 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2446557302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:35.933 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-04-26T04:57:35.934 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:35.936 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export testimg /tmp/rbd_import_export_85090/img2 2024-04-26T04:57:36.206 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:36.209 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export testimg - 2024-04-26T04:57:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:36 smithi195 ceph-mon[31427]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:36 smithi195 ceph-mon[31427]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:57:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:36 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:36.781 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:36 smithi152 ceph-mon[39324]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:36.781 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:36 smithi152 ceph-mon[39324]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:57:36.781 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:36 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:36.781 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:36 smithi152 ceph-mon[27560]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:36.781 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:36 smithi152 ceph-mon[27560]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:57:36.781 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:36 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:36.977 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:36.981 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm testimg 2024-04-26T04:57:37.069 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-04-26T04:57:37.073 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp /tmp/rbd_import_export_85090/img /tmp/rbd_import_export_85090/img2 2024-04-26T04:57:37.677 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp /tmp/rbd_import_export_85090/img /tmp/rbd_import_export_85090/img3 2024-04-26T04:57:38.133 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img2 /tmp/rbd_import_export_85090/img3 2024-04-26T04:57:38.322 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format - testimg 2024-04-26T04:57:38.741 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:38.742 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:38.744 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export testimg /tmp/rbd_import_export_85090/img2 2024-04-26T04:57:38.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:38 smithi195 ceph-mon[31427]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 57 KiB/s wr, 38 op/s 2024-04-26T04:57:38.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:38 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3244044982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:38.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:38 smithi152 ceph-mon[27560]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 57 KiB/s wr, 38 op/s 2024-04-26T04:57:38.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:38 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3244044982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:38.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:38 smithi152 ceph-mon[39324]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 57 KiB/s wr, 38 op/s 2024-04-26T04:57:38.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:38 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3244044982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:39.063 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:39.067 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export testimg - 2024-04-26T04:57:39.877 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:39.881 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm testimg 2024-04-26T04:57:39.972 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-04-26T04:57:39.976 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp /tmp/rbd_import_export_85090/img /tmp/rbd_import_export_85090/img2 2024-04-26T04:57:40.566 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp /tmp/rbd_import_export_85090/img /tmp/rbd_import_export_85090/img3 2024-04-26T04:57:40.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:40 smithi195 ceph-mon[31427]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 57 KiB/s wr, 38 op/s 2024-04-26T04:57:40.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:40 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1343428608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:40.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:40 smithi152 ceph-mon[27560]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 57 KiB/s wr, 38 op/s 2024-04-26T04:57:40.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:40 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1343428608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:40.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:40 smithi152 ceph-mon[39324]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 57 KiB/s wr, 38 op/s 2024-04-26T04:57:40.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:40 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1343428608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:40.988 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img /tmp/rbd_import_export_85090/img2 /tmp/rbd_import_export_85090/img3 2024-04-26T04:57:41.020 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:57:40 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:41.214 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd help export 2024-04-26T04:57:41.214 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep -q export-format 2024-04-26T04:57:41.232 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_85090/img bs=1k count=10 seek=100 2024-04-26T04:57:41.233 INFO:tasks.workunit.client.0.smithi152.stderr:10+0 records in 2024-04-26T04:57:41.233 INFO:tasks.workunit.client.0.smithi152.stderr:10+0 records out 2024-04-26T04:57:41.233 INFO:tasks.workunit.client.0.smithi152.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.6256e-05 s, 155 MB/s 2024-04-26T04:57:41.233 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format /tmp/rbd_import_export_85090/img testimg 2024-04-26T04:57:41.290 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:41.291 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:41.293 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap create testimg@snap 2024-04-26T04:57:42.101 INFO:tasks.workunit.client.0.smithi152.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-26T04:57:42.107 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd image-meta set testimg key1 value1 2024-04-26T04:57:42.155 INFO:tasks.workunit.client.0.smithi152.stderr:++ rbd image-meta list testimg 2024-04-26T04:57:42.192 INFO:tasks.workunit.client.0.smithi152.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-04-26T04:57:42.192 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-26T04:57:42.192 INFO:tasks.workunit.client.0.smithi152.stderr:Key Value 2024-04-26T04:57:42.192 INFO:tasks.workunit.client.0.smithi152.stderr:key1 value1' 2024-04-26T04:57:42.192 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:42.229 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:42.232 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_85090/img_v2 testimg_import 2024-04-26T04:57:42.379 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:42 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:42] "GET /metrics HTTP/1.1" 200 239091 "" "Prometheus/2.43.0" 2024-04-26T04:57:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:42 smithi152 ceph-mon[27560]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:57:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:42 smithi152 ceph-mon[27560]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 66 KiB/s wr, 44 op/s 2024-04-26T04:57:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:42 smithi152 ceph-mon[39324]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:57:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:42 smithi152 ceph-mon[39324]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 66 KiB/s wr, 44 op/s 2024-04-26T04:57:42.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:42 smithi195 ceph-mon[31427]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:57:42.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:42 smithi195 ceph-mon[31427]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 66 KiB/s wr, 44 op/s 2024-04-26T04:57:43.114 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-26T04:57:43.116 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd info testimg_import 2024-04-26T04:57:43.150 INFO:tasks.workunit.client.0.smithi152.stdout:rbd image 'testimg_import': 2024-04-26T04:57:43.150 INFO:tasks.workunit.client.0.smithi152.stdout: size 110 KiB in 1 objects 2024-04-26T04:57:43.150 INFO:tasks.workunit.client.0.smithi152.stdout: order 22 (4 MiB objects) 2024-04-26T04:57:43.150 INFO:tasks.workunit.client.0.smithi152.stdout: snapshot_count: 1 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: id: 3a4142dbc1d4 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: block_name_prefix: rbd_data.3a4142dbc1d4 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: format: 2 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: op_features: 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: flags: 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: create_timestamp: Fri Apr 26 04:57:42 2024 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: access_timestamp: Fri Apr 26 04:57:42 2024 2024-04-26T04:57:43.151 INFO:tasks.workunit.client.0.smithi152.stdout: modify_timestamp: Fri Apr 26 04:57:42 2024 2024-04-26T04:57:43.153 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd info testimg_import@snap 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout:rbd image 'testimg_import': 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: size 110 KiB in 1 objects 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: order 22 (4 MiB objects) 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: snapshot_count: 1 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: id: 3a4142dbc1d4 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: block_name_prefix: rbd_data.3a4142dbc1d4 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: format: 2 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: op_features: 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: flags: 2024-04-26T04:57:43.194 INFO:tasks.workunit.client.0.smithi152.stdout: create_timestamp: Fri Apr 26 04:57:42 2024 2024-04-26T04:57:43.195 INFO:tasks.workunit.client.0.smithi152.stdout: access_timestamp: Fri Apr 26 04:57:42 2024 2024-04-26T04:57:43.195 INFO:tasks.workunit.client.0.smithi152.stdout: modify_timestamp: Fri Apr 26 04:57:42 2024 2024-04-26T04:57:43.195 INFO:tasks.workunit.client.0.smithi152.stdout: protected: False 2024-04-26T04:57:43.197 INFO:tasks.workunit.client.0.smithi152.stderr:++ rbd image-meta list testimg_import 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:Key Value 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:key1 value1' 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 'There is 1 metadatum on this image: 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:Key Value 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:Key Value 2024-04-26T04:57:43.236 INFO:tasks.workunit.client.0.smithi152.stderr:key1 value1' ']' 2024-04-26T04:57:43.237 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export testimg_import /tmp/rbd_import_export_85090/img_import 2024-04-26T04:57:43.270 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete...done. 2024-04-26T04:57:43.273 INFO:tasks.workunit.client.0.smithi152.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_85090/img /tmp/rbd_import_export_85090/img_import 2024-04-26T04:57:43.274 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp -l /tmp/rbd_import_export_85090/img /tmp/rbd_import_export_85090/img_import 2024-04-26T04:57:43.274 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/img --format %b 2024-04-26T04:57:43.275 INFO:tasks.workunit.client.0.smithi152.stderr:+ origsize=24 2024-04-26T04:57:43.275 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/img_import --format %b 2024-04-26T04:57:43.277 INFO:tasks.workunit.client.0.smithi152.stderr:+ exportsize=224 2024-04-26T04:57:43.277 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=200 2024-04-26T04:57:43.277 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=200 2024-04-26T04:57:43.277 INFO:tasks.workunit.client.0.smithi152.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-04-26T04:57:43.277 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_85090/img_snap 2024-04-26T04:57:43.313 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete...done. 2024-04-26T04:57:43.316 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_85090/img_snap_import 2024-04-26T04:57:43.353 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete...done. 2024-04-26T04:57:43.355 INFO:tasks.workunit.client.0.smithi152.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_85090/img_snap /tmp/rbd_import_export_85090/img_snap_import 2024-04-26T04:57:43.355 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp -l /tmp/rbd_import_export_85090/img_snap /tmp/rbd_import_export_85090/img_snap_import 2024-04-26T04:57:43.356 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/img_snap --format %b 2024-04-26T04:57:43.357 INFO:tasks.workunit.client.0.smithi152.stderr:+ origsize=224 2024-04-26T04:57:43.357 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/img_snap_import --format %b 2024-04-26T04:57:43.358 INFO:tasks.workunit.client.0.smithi152.stderr:+ exportsize=224 2024-04-26T04:57:43.359 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:43.359 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:43.359 INFO:tasks.workunit.client.0.smithi152.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-26T04:57:43.359 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:43.359 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img_import 2024-04-26T04:57:43.360 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img_snap 2024-04-26T04:57:43.360 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img_snap_import 2024-04-26T04:57:43.361 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap rm testimg_import@snap 2024-04-26T04:57:43.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:43 smithi152 ceph-mon[27560]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:57:43.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:43 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/358675931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:43.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:43 smithi152 ceph-mon[39324]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:57:43.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:43 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/358675931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:43.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:43 smithi195 ceph-mon[31427]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:57:43.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:43 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/358675931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:44.112 INFO:tasks.workunit.client.0.smithi152.stderr: Removing snap: 100% complete...done. 2024-04-26T04:57:44.118 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg_import 2024-04-26T04:57:44.201 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:44.204 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap rm testimg@snap 2024-04-26T04:57:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:44 smithi152 ceph-mon[27560]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:57:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:44 smithi152 ceph-mon[27560]: pgmap v105: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 86 KiB/s wr, 138 op/s 2024-04-26T04:57:44.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:44 smithi152 ceph-mon[39324]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:57:44.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:44 smithi152 ceph-mon[39324]: pgmap v105: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 86 KiB/s wr, 138 op/s 2024-04-26T04:57:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:44 smithi195 ceph-mon[31427]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:57:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:44 smithi195 ceph-mon[31427]: pgmap v105: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 86 KiB/s wr, 138 op/s 2024-04-26T04:57:45.111 INFO:tasks.workunit.client.0.smithi152.stderr: Removing snap: 100% complete...done. 2024-04-26T04:57:45.120 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm testimg 2024-04-26T04:57:45.212 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:45.215 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --order 20 /tmp/rbd_import_export_85090/img testimg 2024-04-26T04:57:45.271 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:45.272 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --order is deprecated, use --object-size 2024-04-26T04:57:45.274 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:45.312 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:45.315 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_85090/img_v2 testimg_import 2024-04-26T04:57:45.375 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-26T04:57:45.379 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd info testimg_import 2024-04-26T04:57:45.379 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep order 2024-04-26T04:57:45.379 INFO:tasks.workunit.client.0.smithi152.stderr:+ awk '{print $2}' 2024-04-26T04:57:45.379 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep 20 2024-04-26T04:57:45.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:45 smithi152 ceph-mon[27560]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:57:45.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:45 smithi152 ceph-mon[39324]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:57:45.414 INFO:tasks.workunit.client.0.smithi152.stdout:20 2024-04-26T04:57:45.414 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:45.415 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg_import 2024-04-26T04:57:45.500 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:45.503 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg 2024-04-26T04:57:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:45 smithi195 ceph-mon[31427]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:57:45.590 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:45.593 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_85090/img testimg 2024-04-26T04:57:45.643 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:45.646 INFO:tasks.workunit.client.0.smithi152.stderr:++ rbd info testimg 2024-04-26T04:57:45.646 INFO:tasks.workunit.client.0.smithi152.stderr:++ grep features 2024-04-26T04:57:45.682 INFO:tasks.workunit.client.0.smithi152.stderr:+ FEATURES_BEFORE=' features: layering 2024-04-26T04:57:45.682 INFO:tasks.workunit.client.0.smithi152.stderr: op_features: ' 2024-04-26T04:57:45.682 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:45.722 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:45.725 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_85090/img_v2 testimg_import 2024-04-26T04:57:45.777 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-26T04:57:45.781 INFO:tasks.workunit.client.0.smithi152.stderr:++ rbd info testimg_import 2024-04-26T04:57:45.781 INFO:tasks.workunit.client.0.smithi152.stderr:++ grep features 2024-04-26T04:57:45.818 INFO:tasks.workunit.client.0.smithi152.stderr:+ FEATURES_AFTER=' features: layering 2024-04-26T04:57:45.819 INFO:tasks.workunit.client.0.smithi152.stderr: op_features: ' 2024-04-26T04:57:45.819 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' ' features: layering 2024-04-26T04:57:45.819 INFO:tasks.workunit.client.0.smithi152.stderr: op_features: ' '!=' ' features: layering 2024-04-26T04:57:45.819 INFO:tasks.workunit.client.0.smithi152.stderr: op_features: ' ']' 2024-04-26T04:57:45.819 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:45.819 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg_import 2024-04-26T04:57:45.889 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:45.892 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg 2024-04-26T04:57:45.960 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:45.962 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_85090/img testimg 2024-04-26T04:57:46.020 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:46.022 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:46.061 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:46.065 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_85090/img_v2 testimg_import 2024-04-26T04:57:46.129 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-26T04:57:46.132 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd info testimg_import 2024-04-26T04:57:46.132 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep 'stripe unit' 2024-04-26T04:57:46.132 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep -Ei '(4 KiB|4096)' 2024-04-26T04:57:46.169 INFO:tasks.workunit.client.0.smithi152.stdout: stripe unit: 4 KiB 2024-04-26T04:57:46.170 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd info testimg_import 2024-04-26T04:57:46.170 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep 'stripe count' 2024-04-26T04:57:46.170 INFO:tasks.workunit.client.0.smithi152.stderr:+ awk '{print $3}' 2024-04-26T04:57:46.170 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep 1000 2024-04-26T04:57:46.205 INFO:tasks.workunit.client.0.smithi152.stdout:1000 2024-04-26T04:57:46.205 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/img_v2 2024-04-26T04:57:46.206 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg_import 2024-04-26T04:57:46.290 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-04-26T04:57:46.293 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg 2024-04-26T04:57:46.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:46 smithi152 ceph-mon[27560]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:57:46.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:46 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/298877433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:46 smithi152 ceph-mon[27560]: pgmap v108: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 371 KiB/s rd, 129 KiB/s wr, 208 op/s 2024-04-26T04:57:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:46 smithi152 ceph-mon[39324]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:57:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:46 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/298877433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:46 smithi152 ceph-mon[39324]: pgmap v108: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 371 KiB/s rd, 129 KiB/s wr, 208 op/s 2024-04-26T04:57:46.380 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-04-26T04:57:46.382 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_85090/img testimg 2024-04-26T04:57:46.434 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:46.438 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap create testimg@snap1 2024-04-26T04:57:46.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:46 smithi195 ceph-mon[31427]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:57:46.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:46 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/298877433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:46.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:46 smithi195 ceph-mon[31427]: pgmap v108: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 371 KiB/s rd, 129 KiB/s wr, 208 op/s 2024-04-26T04:57:47.128 INFO:tasks.workunit.client.0.smithi152.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-26T04:57:47.135 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap create testimg@snap2 2024-04-26T04:57:48.133 INFO:tasks.workunit.client.0.smithi152.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-26T04:57:48.139 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap protect testimg@snap2 2024-04-26T04:57:48.184 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_85090/snap_protect 2024-04-26T04:57:48.224 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:48.227 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_85090/snap_protect testimg_import 2024-04-26T04:57:48.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:48 smithi152 ceph-mon[27560]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:57:48.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:48 smithi152 ceph-mon[27560]: pgmap v110: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 428 KiB/s rd, 45 KiB/s wr, 563 op/s 2024-04-26T04:57:48.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:48 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1333854126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:48.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:48 smithi152 ceph-mon[39324]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:57:48.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:48 smithi152 ceph-mon[39324]: pgmap v110: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 428 KiB/s rd, 45 KiB/s wr, 563 op/s 2024-04-26T04:57:48.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:48 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1333854126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:48 smithi195 ceph-mon[31427]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:57:48.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:48 smithi195 ceph-mon[31427]: pgmap v110: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 428 KiB/s rd, 45 KiB/s wr, 563 op/s 2024-04-26T04:57:48.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:48 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1333854126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:49.451 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:49 smithi152 ceph-mon[27560]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:57:49.451 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:49 smithi152 ceph-mon[39324]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:57:49.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:49 smithi195 ceph-mon[31427]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:57:50.167 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-26T04:57:50.172 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd info testimg_import@snap1 2024-04-26T04:57:50.172 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep 'protected: False' 2024-04-26T04:57:50.211 INFO:tasks.workunit.client.0.smithi152.stdout: protected: False 2024-04-26T04:57:50.212 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd info testimg_import@snap2 2024-04-26T04:57:50.212 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep 'protected: True' 2024-04-26T04:57:50.251 INFO:tasks.workunit.client.0.smithi152.stdout: protected: True 2024-04-26T04:57:50.251 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/snap_protect 2024-04-26T04:57:50.252 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap unprotect testimg@snap2 2024-04-26T04:57:50.295 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap unprotect testimg_import@snap2 2024-04-26T04:57:50.338 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap purge testimg 2024-04-26T04:57:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:50 smithi195 ceph-mon[31427]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:57:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:50 smithi195 ceph-mon[31427]: pgmap v113: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 429 KiB/s rd, 45 KiB/s wr, 564 op/s 2024-04-26T04:57:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:50 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3131907269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:50.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:50 smithi152 ceph-mon[27560]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:57:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:50 smithi152 ceph-mon[27560]: pgmap v113: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 429 KiB/s rd, 45 KiB/s wr, 564 op/s 2024-04-26T04:57:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:50 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3131907269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:50.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:50 smithi152 ceph-mon[39324]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:57:50.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:50 smithi152 ceph-mon[39324]: pgmap v113: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 429 KiB/s rd, 45 KiB/s wr, 564 op/s 2024-04-26T04:57:50.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:50 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3131907269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:51.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:57:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:51.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:51 smithi195 ceph-mon[31427]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:57:51.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:51 smithi195 ceph-mon[31427]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:57:51.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:51 smithi152 ceph-mon[27560]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:57:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:51 smithi152 ceph-mon[27560]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:57:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:51 smithi152 ceph-mon[39324]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:57:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:51 smithi152 ceph-mon[39324]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:57:52.099 INFO:tasks.workunit.client.0.smithi152.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-04-26T04:57:52.106 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd snap purge testimg_import 2024-04-26T04:57:52.483 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:52 smithi152 ceph-mon[27560]: pgmap v116: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:52.483 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:52 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:52.483 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:52 smithi152 ceph-mon[27560]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:57:52.484 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:57:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:57:52] "GET /metrics HTTP/1.1" 200 239815 "" "Prometheus/2.43.0" 2024-04-26T04:57:52.484 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:52 smithi152 ceph-mon[39324]: pgmap v116: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:52.484 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:52 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:52.484 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:52 smithi152 ceph-mon[39324]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:57:52.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:52 smithi195 ceph-mon[31427]: pgmap v116: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:52.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:52 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:52.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:52 smithi195 ceph-mon[31427]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:57:53.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:53 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2537455496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:53.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:53 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2537455496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:53.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:53 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2537455496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:54.167 INFO:tasks.workunit.client.0.smithi152.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-04-26T04:57:54.173 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg 2024-04-26T04:57:54.262 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:54.264 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd remove testimg_import 2024-04-26T04:57:54.346 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:54.349 INFO:tasks.workunit.client.0.smithi152.stderr:+ tiered=0 2024-04-26T04:57:54.349 INFO:tasks.workunit.client.0.smithi152.stderr:+ ceph osd dump 2024-04-26T04:57:54.349 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep '^pool' 2024-04-26T04:57:54.349 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep ''\''rbd'\''' 2024-04-26T04:57:54.349 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep tier 2024-04-26T04:57:54.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[27560]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:57:54.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[27560]: pgmap v119: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[39324]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[39324]: pgmap v119: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:54.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:54 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:54.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:54 smithi195 ceph-mon[31427]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:57:54.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:54 smithi195 ceph-mon[31427]: pgmap v119: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-04-26T04:57:54.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:54 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:54.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:54 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:54.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:54 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:57:54.728 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_85090/sparse1 2024-04-26T04:57:54.732 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records in 2024-04-26T04:57:54.732 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records out 2024-04-26T04:57:54.732 INFO:tasks.workunit.client.0.smithi152.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00334885 s, 313 MB/s 2024-04-26T04:57:54.732 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_85090/sparse2 2024-04-26T04:57:54.736 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records in 2024-04-26T04:57:54.736 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records out 2024-04-26T04:57:54.736 INFO:tasks.workunit.client.0.smithi152.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0032952 s, 318 MB/s 2024-04-26T04:57:54.736 INFO:tasks.workunit.client.0.smithi152.stderr:+ truncate /tmp/rbd_import_export_85090/sparse2 -s 2M 2024-04-26T04:57:54.745 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse1 2024-04-26T04:57:54.782 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 0% complete...failed. 2024-04-26T04:57:54.783 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: delete error: (2) No such file or directory 2024-04-26T04:57:54.785 INFO:tasks.workunit.client.0.smithi152.stderr:+ true 2024-04-26T04:57:54.785 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_85090/sparse1 2024-04-26T04:57:54.845 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-04-26T04:57:54.845 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:54.845 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --order is deprecated, use --object-size 2024-04-26T04:57:54.848 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd ls -l 2024-04-26T04:57:54.848 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep sparse1 2024-04-26T04:57:54.848 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-04-26T04:57:54.883 INFO:tasks.workunit.client.0.smithi152.stdout:sparse1 2 MiB 2 2024-04-26T04:57:54.884 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects sparse1 2024-04-26T04:57:54.884 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse1 2024-04-26T04:57:54.884 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse1 2024-04-26T04:57:54.884 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep block_name_prefix 2024-04-26T04:57:54.884 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk '{print $NF;}' 2024-04-26T04:57:54.918 INFO:tasks.workunit.client.0.smithi152.stderr:++ prefix=rbd_data.3b7f7f8710c8 2024-04-26T04:57:54.919 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep rbd_data.3b7f7f8710c8 2024-04-26T04:57:54.919 INFO:tasks.workunit.client.0.smithi152.stderr:++++ get_image_data_pool sparse1 2024-04-26T04:57:54.919 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sed -e 's/rbd_data.3b7f7f8710c8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-26T04:57:54.919 INFO:tasks.workunit.client.0.smithi152.stderr:++++ image=sparse1 2024-04-26T04:57:54.919 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sort -u 2024-04-26T04:57:54.920 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ rbd info sparse1 2024-04-26T04:57:54.920 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ grep 'data_pool: ' 2024-04-26T04:57:54.920 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ awk -F: '{ print $NF }' 2024-04-26T04:57:54.954 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool= 2024-04-26T04:57:54.955 INFO:tasks.workunit.client.0.smithi152.stderr:++++ '[' -z ']' 2024-04-26T04:57:54.955 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool=rbd 2024-04-26T04:57:54.955 INFO:tasks.workunit.client.0.smithi152.stderr:++++ echo rbd 2024-04-26T04:57:54.955 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rados ls -p rbd 2024-04-26T04:57:54.986 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects=1 2024-04-26T04:57:54.986 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo 1 2024-04-26T04:57:54.986 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-04-26T04:57:54.986 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export sparse1 /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.020 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:55.023 INFO:tasks.workunit.client.0.smithi152.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_85090/sparse1 /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.023 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp -l /tmp/rbd_import_export_85090/sparse1 /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.025 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse1 --format %b 2024-04-26T04:57:55.026 INFO:tasks.workunit.client.0.smithi152.stderr:+ origsize=2048 2024-04-26T04:57:55.026 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse1.out --format %b 2024-04-26T04:57:55.027 INFO:tasks.workunit.client.0.smithi152.stderr:+ exportsize=2048 2024-04-26T04:57:55.027 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:55.027 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:55.027 INFO:tasks.workunit.client.0.smithi152.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-26T04:57:55.027 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.028 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse1 2024-04-26T04:57:55.108 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-04-26T04:57:55.111 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse2 2024-04-26T04:57:55.145 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 0% complete...failed. 2024-04-26T04:57:55.145 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: delete error: (2) No such file or directory 2024-04-26T04:57:55.148 INFO:tasks.workunit.client.0.smithi152.stderr:+ true 2024-04-26T04:57:55.148 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_85090/sparse2 2024-04-26T04:57:55.203 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-04-26T04:57:55.203 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:55.203 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --order is deprecated, use --object-size 2024-04-26T04:57:55.206 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd ls -l 2024-04-26T04:57:55.206 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep sparse2 2024-04-26T04:57:55.206 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-04-26T04:57:55.240 INFO:tasks.workunit.client.0.smithi152.stdout:sparse2 2 MiB 2 2024-04-26T04:57:55.240 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects sparse2 2024-04-26T04:57:55.241 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse2 2024-04-26T04:57:55.241 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse2 2024-04-26T04:57:55.241 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep block_name_prefix 2024-04-26T04:57:55.241 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk '{print $NF;}' 2024-04-26T04:57:55.275 INFO:tasks.workunit.client.0.smithi152.stderr:++ prefix=rbd_data.3bb2d88a10be 2024-04-26T04:57:55.275 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep rbd_data.3bb2d88a10be 2024-04-26T04:57:55.275 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sed -e 's/rbd_data.3bb2d88a10be\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-26T04:57:55.275 INFO:tasks.workunit.client.0.smithi152.stderr:++++ get_image_data_pool sparse2 2024-04-26T04:57:55.276 INFO:tasks.workunit.client.0.smithi152.stderr:++++ image=sparse2 2024-04-26T04:57:55.276 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sort -u 2024-04-26T04:57:55.276 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ rbd info sparse2 2024-04-26T04:57:55.276 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ grep 'data_pool: ' 2024-04-26T04:57:55.276 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ awk -F: '{ print $NF }' 2024-04-26T04:57:55.310 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool= 2024-04-26T04:57:55.310 INFO:tasks.workunit.client.0.smithi152.stderr:++++ '[' -z ']' 2024-04-26T04:57:55.310 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool=rbd 2024-04-26T04:57:55.310 INFO:tasks.workunit.client.0.smithi152.stderr:++++ echo rbd 2024-04-26T04:57:55.310 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rados ls -p rbd 2024-04-26T04:57:55.341 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects=0 2024-04-26T04:57:55.341 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo 0 2024-04-26T04:57:55.341 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-04-26T04:57:55.342 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export sparse2 /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:55.375 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:55.378 INFO:tasks.workunit.client.0.smithi152.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_85090/sparse2 /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:55.378 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp -l /tmp/rbd_import_export_85090/sparse2 /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:55.380 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse2 --format %b 2024-04-26T04:57:55.381 INFO:tasks.workunit.client.0.smithi152.stderr:+ origsize=2048 2024-04-26T04:57:55.381 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse2.out --format %b 2024-04-26T04:57:55.382 INFO:tasks.workunit.client.0.smithi152.stderr:+ exportsize=2048 2024-04-26T04:57:55.382 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:55.382 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:55.382 INFO:tasks.workunit.client.0.smithi152.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-26T04:57:55.382 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:55.382 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse2 2024-04-26T04:57:55.459 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-04-26T04:57:55.462 INFO:tasks.workunit.client.0.smithi152.stderr:+ truncate /tmp/rbd_import_export_85090/sparse1 -s 10M 2024-04-26T04:57:55.463 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format --order 20 - sparse1 2024-04-26T04:57:55.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:57:55.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:57:55.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:57:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:57:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi195 2024-04-26T04:57:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1390247195' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-26T04:57:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1390247195' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-26T04:57:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2950113735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:55.542 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:55.543 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:55.543 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --order is deprecated, use --object-size 2024-04-26T04:57:55.546 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd ls -l 2024-04-26T04:57:55.546 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep sparse1 2024-04-26T04:57:55.546 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-04-26T04:57:55.580 INFO:tasks.workunit.client.0.smithi152.stdout:sparse1 10 MiB 2 2024-04-26T04:57:55.581 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects sparse1 2024-04-26T04:57:55.581 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse1 2024-04-26T04:57:55.581 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse1 2024-04-26T04:57:55.581 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep block_name_prefix 2024-04-26T04:57:55.581 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk '{print $NF;}' 2024-04-26T04:57:55.616 INFO:tasks.workunit.client.0.smithi152.stderr:++ prefix=rbd_data.3bdf4803bfda 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep rbd_data.3bdf4803bfda 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:++++ get_image_data_pool sparse1 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sed -e 's/rbd_data.3bdf4803bfda\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:++++ image=sparse1 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sort -u 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ rbd info sparse1 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ grep 'data_pool: ' 2024-04-26T04:57:55.617 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ awk -F: '{ print $NF }' 2024-04-26T04:57:55.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi195 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1390247195' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1390247195' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2950113735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:57:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:57:55.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:57:55.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi195 2024-04-26T04:57:55.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1390247195' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-26T04:57:55.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1390247195' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-26T04:57:55.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2950113735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:55.652 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool= 2024-04-26T04:57:55.652 INFO:tasks.workunit.client.0.smithi152.stderr:++++ '[' -z ']' 2024-04-26T04:57:55.653 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool=rbd 2024-04-26T04:57:55.653 INFO:tasks.workunit.client.0.smithi152.stderr:++++ echo rbd 2024-04-26T04:57:55.653 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rados ls -p rbd 2024-04-26T04:57:55.683 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects=1 2024-04-26T04:57:55.683 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo 1 2024-04-26T04:57:55.683 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-04-26T04:57:55.684 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export sparse1 /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.719 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:55.723 INFO:tasks.workunit.client.0.smithi152.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_85090/sparse1 /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.723 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp -l /tmp/rbd_import_export_85090/sparse1 /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.730 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse1 --format %b 2024-04-26T04:57:55.730 INFO:tasks.workunit.client.0.smithi152.stderr:+ origsize=2048 2024-04-26T04:57:55.730 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse1.out --format %b 2024-04-26T04:57:55.732 INFO:tasks.workunit.client.0.smithi152.stderr:+ exportsize=2048 2024-04-26T04:57:55.732 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:55.732 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:55.732 INFO:tasks.workunit.client.0.smithi152.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-26T04:57:55.732 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/sparse1.out 2024-04-26T04:57:55.733 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse1 2024-04-26T04:57:55.812 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-04-26T04:57:55.815 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_85090/sparse2 oflag=append conv=notrunc 2024-04-26T04:57:55.822 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records in 2024-04-26T04:57:55.822 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records out 2024-04-26T04:57:55.822 INFO:tasks.workunit.client.0.smithi152.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00613596 s, 342 MB/s 2024-04-26T04:57:55.822 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format --order 20 - sparse2 2024-04-26T04:57:55.896 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:55.896 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:55.897 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --order is deprecated, use --object-size 2024-04-26T04:57:55.899 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd ls -l 2024-04-26T04:57:55.899 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep sparse2 2024-04-26T04:57:55.899 INFO:tasks.workunit.client.0.smithi152.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-04-26T04:57:55.933 INFO:tasks.workunit.client.0.smithi152.stdout:sparse2 4 MiB 2 2024-04-26T04:57:55.934 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects sparse2 2024-04-26T04:57:55.934 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse2 2024-04-26T04:57:55.934 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse2 2024-04-26T04:57:55.934 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep block_name_prefix 2024-04-26T04:57:55.934 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk '{print $NF;}' 2024-04-26T04:57:55.968 INFO:tasks.workunit.client.0.smithi152.stderr:++ prefix=rbd_data.62b7ee76f54c 2024-04-26T04:57:55.969 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep rbd_data.62b7ee76f54c 2024-04-26T04:57:55.969 INFO:tasks.workunit.client.0.smithi152.stderr:++++ get_image_data_pool sparse2 2024-04-26T04:57:55.969 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sed -e 's/rbd_data.62b7ee76f54c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-26T04:57:55.970 INFO:tasks.workunit.client.0.smithi152.stderr:++++ image=sparse2 2024-04-26T04:57:55.970 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sort -u 2024-04-26T04:57:55.970 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ rbd info sparse2 2024-04-26T04:57:55.970 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ grep 'data_pool: ' 2024-04-26T04:57:55.970 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ awk -F: '{ print $NF }' 2024-04-26T04:57:56.004 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool= 2024-04-26T04:57:56.004 INFO:tasks.workunit.client.0.smithi152.stderr:++++ '[' -z ']' 2024-04-26T04:57:56.004 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool=rbd 2024-04-26T04:57:56.004 INFO:tasks.workunit.client.0.smithi152.stderr:++++ echo rbd 2024-04-26T04:57:56.004 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rados ls -p rbd 2024-04-26T04:57:56.035 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects='0 2024-04-26T04:57:56.036 INFO:tasks.workunit.client.0.smithi152.stderr:2 2024-04-26T04:57:56.036 INFO:tasks.workunit.client.0.smithi152.stderr:3' 2024-04-26T04:57:56.036 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo 0 2 3 2024-04-26T04:57:56.036 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-04-26T04:57:56.036 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export sparse2 /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:56.071 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-04-26T04:57:56.074 INFO:tasks.workunit.client.0.smithi152.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_85090/sparse2 /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:56.074 INFO:tasks.workunit.client.0.smithi152.stderr:+ cmp -l /tmp/rbd_import_export_85090/sparse2 /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:56.077 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse2 --format %b 2024-04-26T04:57:56.078 INFO:tasks.workunit.client.0.smithi152.stderr:+ origsize=6144 2024-04-26T04:57:56.078 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse2.out --format %b 2024-04-26T04:57:56.079 INFO:tasks.workunit.client.0.smithi152.stderr:+ exportsize=6144 2024-04-26T04:57:56.079 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:56.079 INFO:tasks.workunit.client.0.smithi152.stderr:+ difference=0 2024-04-26T04:57:56.079 INFO:tasks.workunit.client.0.smithi152.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-26T04:57:56.079 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/sparse2.out 2024-04-26T04:57:56.080 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse2 2024-04-26T04:57:56.173 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-04-26T04:57:56.176 INFO:tasks.workunit.client.0.smithi152.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-04-26T04:57:56.176 INFO:tasks.workunit.client.0.smithi152.stdout:partially-sparse file imports to partially-sparse image 2024-04-26T04:57:56.177 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_85090/sparse1 sparse 2024-04-26T04:57:56.235 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-04-26T04:57:56.235 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:56.235 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --order is deprecated, use --object-size 2024-04-26T04:57:56.238 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects sparse 2024-04-26T04:57:56.238 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse 2024-04-26T04:57:56.239 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse 2024-04-26T04:57:56.239 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep block_name_prefix 2024-04-26T04:57:56.239 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk '{print $NF;}' 2024-04-26T04:57:56.273 INFO:tasks.workunit.client.0.smithi152.stderr:++ prefix=rbd_data.62e416fa13cb 2024-04-26T04:57:56.274 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep rbd_data.62e416fa13cb 2024-04-26T04:57:56.274 INFO:tasks.workunit.client.0.smithi152.stderr:++++ get_image_data_pool sparse 2024-04-26T04:57:56.275 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sed -e 's/rbd_data.62e416fa13cb\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-26T04:57:56.275 INFO:tasks.workunit.client.0.smithi152.stderr:++++ image=sparse 2024-04-26T04:57:56.275 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sort -u 2024-04-26T04:57:56.275 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ rbd info sparse 2024-04-26T04:57:56.275 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ grep 'data_pool: ' 2024-04-26T04:57:56.275 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ awk -F: '{ print $NF }' 2024-04-26T04:57:56.309 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool= 2024-04-26T04:57:56.309 INFO:tasks.workunit.client.0.smithi152.stderr:++++ '[' -z ']' 2024-04-26T04:57:56.309 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool=rbd 2024-04-26T04:57:56.309 INFO:tasks.workunit.client.0.smithi152.stderr:++++ echo rbd 2024-04-26T04:57:56.309 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rados ls -p rbd 2024-04-26T04:57:56.340 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects=1 2024-04-26T04:57:56.341 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo 1 2024-04-26T04:57:56.341 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-04-26T04:57:56.341 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse 2024-04-26T04:57:56.422 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-04-26T04:57:56.424 INFO:tasks.workunit.client.0.smithi152.stdout:zeros import through stdin to sparse image 2024-04-26T04:57:56.425 INFO:tasks.workunit.client.0.smithi152.stderr:+ echo 'zeros import through stdin to sparse image' 2024-04-26T04:57:56.425 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/dev/zero bs=1M count=4 2024-04-26T04:57:56.425 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd import --new-format - sparse 2024-04-26T04:57:56.469 INFO:tasks.workunit.client.0.smithi152.stderr:4+0 records in 2024-04-26T04:57:56.469 INFO:tasks.workunit.client.0.smithi152.stderr:4+0 records out 2024-04-26T04:57:56.470 INFO:tasks.workunit.client.0.smithi152.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0436762 s, 96.0 MB/s 2024-04-26T04:57:56.478 INFO:tasks.workunit.client.0.smithi152.stderr: Importing image: 100% complete...done. 2024-04-26T04:57:56.478 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:56.481 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects sparse 2024-04-26T04:57:56.481 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse 2024-04-26T04:57:56.481 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse 2024-04-26T04:57:56.481 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep block_name_prefix 2024-04-26T04:57:56.481 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk '{print $NF;}' 2024-04-26T04:57:56.514 INFO:tasks.workunit.client.0.smithi152.stderr:++ prefix=rbd_data.6305b53cee60 2024-04-26T04:57:56.515 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep rbd_data.6305b53cee60 2024-04-26T04:57:56.515 INFO:tasks.workunit.client.0.smithi152.stderr:++++ get_image_data_pool sparse 2024-04-26T04:57:56.515 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sed -e 's/rbd_data.6305b53cee60\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-26T04:57:56.515 INFO:tasks.workunit.client.0.smithi152.stderr:++++ image=sparse 2024-04-26T04:57:56.516 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sort -u 2024-04-26T04:57:56.516 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ rbd info sparse 2024-04-26T04:57:56.516 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ grep 'data_pool: ' 2024-04-26T04:57:56.516 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ awk -F: '{ print $NF }' 2024-04-26T04:57:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:56 smithi195 ceph-mon[31427]: pgmap v121: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 122 KiB/s rd, 11 KiB/s wr, 162 op/s 2024-04-26T04:57:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:56 smithi195 ceph-mon[31427]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:57:56.549 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool= 2024-04-26T04:57:56.549 INFO:tasks.workunit.client.0.smithi152.stderr:++++ '[' -z ']' 2024-04-26T04:57:56.550 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool=rbd 2024-04-26T04:57:56.550 INFO:tasks.workunit.client.0.smithi152.stderr:++++ echo rbd 2024-04-26T04:57:56.550 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rados ls -p rbd 2024-04-26T04:57:56.580 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects= 2024-04-26T04:57:56.581 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo 2024-04-26T04:57:56.581 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-04-26T04:57:56.581 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse 2024-04-26T04:57:56.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:56 smithi152 ceph-mon[27560]: pgmap v121: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 122 KiB/s rd, 11 KiB/s wr, 162 op/s 2024-04-26T04:57:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:56 smithi152 ceph-mon[27560]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:57:56.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:56 smithi152 ceph-mon[39324]: pgmap v121: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 122 KiB/s rd, 11 KiB/s wr, 162 op/s 2024-04-26T04:57:56.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:56 smithi152 ceph-mon[39324]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:57:56.654 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:56.657 INFO:tasks.workunit.client.0.smithi152.stdout:zeros export to sparse file 2024-04-26T04:57:56.657 INFO:tasks.workunit.client.0.smithi152.stderr:+ echo 'zeros export to sparse file' 2024-04-26T04:57:56.658 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd create --new-format sparse --size 4 2024-04-26T04:57:56.694 INFO:tasks.workunit.client.0.smithi152.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-26T04:57:56.697 INFO:tasks.workunit.client.0.smithi152.stderr:++ grep block_name_prefix 2024-04-26T04:57:56.697 INFO:tasks.workunit.client.0.smithi152.stderr:++ rbd info sparse 2024-04-26T04:57:56.697 INFO:tasks.workunit.client.0.smithi152.stderr:++ awk '{print $NF;}' 2024-04-26T04:57:56.730 INFO:tasks.workunit.client.0.smithi152.stderr:+ prefix=rbd_data.3c7bd93bb021 2024-04-26T04:57:56.731 INFO:tasks.workunit.client.0.smithi152.stderr:+ dd if=/dev/zero bs=4M count=1 2024-04-26T04:57:56.731 INFO:tasks.workunit.client.0.smithi152.stderr:++ get_image_data_pool sparse 2024-04-26T04:57:56.731 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse 2024-04-26T04:57:56.731 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse 2024-04-26T04:57:56.731 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep 'data_pool: ' 2024-04-26T04:57:56.731 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk -F: '{ print $NF }' 2024-04-26T04:57:56.765 INFO:tasks.workunit.client.0.smithi152.stderr:++ data_pool= 2024-04-26T04:57:56.765 INFO:tasks.workunit.client.0.smithi152.stderr:++ '[' -z ']' 2024-04-26T04:57:56.765 INFO:tasks.workunit.client.0.smithi152.stderr:++ data_pool=rbd 2024-04-26T04:57:56.765 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo rbd 2024-04-26T04:57:56.765 INFO:tasks.workunit.client.0.smithi152.stderr:+ rados -p rbd put rbd_data.3c7bd93bb021.000000000000 - 2024-04-26T04:57:56.793 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records in 2024-04-26T04:57:56.793 INFO:tasks.workunit.client.0.smithi152.stderr:1+0 records out 2024-04-26T04:57:56.793 INFO:tasks.workunit.client.0.smithi152.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0620085 s, 67.6 MB/s 2024-04-26T04:57:56.811 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects sparse 2024-04-26T04:57:56.811 INFO:tasks.workunit.client.0.smithi152.stderr:++ image=sparse 2024-04-26T04:57:56.812 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rbd info sparse 2024-04-26T04:57:56.812 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep block_name_prefix 2024-04-26T04:57:56.812 INFO:tasks.workunit.client.0.smithi152.stderr:+++ awk '{print $NF;}' 2024-04-26T04:57:56.845 INFO:tasks.workunit.client.0.smithi152.stderr:++ prefix=rbd_data.3c7bd93bb021 2024-04-26T04:57:56.846 INFO:tasks.workunit.client.0.smithi152.stderr:+++ grep rbd_data.3c7bd93bb021 2024-04-26T04:57:56.846 INFO:tasks.workunit.client.0.smithi152.stderr:++++ get_image_data_pool sparse 2024-04-26T04:57:56.846 INFO:tasks.workunit.client.0.smithi152.stderr:++++ image=sparse 2024-04-26T04:57:56.846 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sed -e 's/rbd_data.3c7bd93bb021\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-26T04:57:56.847 INFO:tasks.workunit.client.0.smithi152.stderr:+++ sort -u 2024-04-26T04:57:56.847 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ rbd info sparse 2024-04-26T04:57:56.847 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ grep 'data_pool: ' 2024-04-26T04:57:56.847 INFO:tasks.workunit.client.0.smithi152.stderr:+++++ awk -F: '{ print $NF }' 2024-04-26T04:57:56.882 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool= 2024-04-26T04:57:56.882 INFO:tasks.workunit.client.0.smithi152.stderr:++++ '[' -z ']' 2024-04-26T04:57:56.882 INFO:tasks.workunit.client.0.smithi152.stderr:++++ data_pool=rbd 2024-04-26T04:57:56.882 INFO:tasks.workunit.client.0.smithi152.stderr:++++ echo rbd 2024-04-26T04:57:56.882 INFO:tasks.workunit.client.0.smithi152.stderr:+++ rados ls -p rbd 2024-04-26T04:57:56.913 INFO:tasks.workunit.client.0.smithi152.stderr:++ objects=0 2024-04-26T04:57:56.913 INFO:tasks.workunit.client.0.smithi152.stderr:++ echo 0 2024-04-26T04:57:56.913 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-04-26T04:57:56.913 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/sparse 2024-04-26T04:57:56.914 INFO:tasks.workunit.client.0.smithi152.stderr:rm: cannot remove '/tmp/rbd_import_export_85090/sparse': No such file or directory 2024-04-26T04:57:56.914 INFO:tasks.workunit.client.0.smithi152.stderr:+ true 2024-04-26T04:57:56.914 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd export sparse /tmp/rbd_import_export_85090/sparse 2024-04-26T04:57:56.948 INFO:tasks.workunit.client.0.smithi152.stderr: Exporting image: 100% complete...done. 2024-04-26T04:57:56.953 INFO:tasks.workunit.client.0.smithi152.stderr:++ stat /tmp/rbd_import_export_85090/sparse --format=%b 2024-04-26T04:57:56.954 INFO:tasks.workunit.client.0.smithi152.stderr:+ '[' 0 = 0 ']' 2024-04-26T04:57:56.954 INFO:tasks.workunit.client.0.smithi152.stderr:+ rbd rm sparse 2024-04-26T04:57:57.031 INFO:tasks.workunit.client.0.smithi152.stderr: Removing image: 100% complete...done. 2024-04-26T04:57:57.034 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm /tmp/rbd_import_export_85090/sparse /tmp/rbd_import_export_85090/sparse1 /tmp/rbd_import_export_85090/sparse2 /tmp/rbd_import_export_85090/sparse3 2024-04-26T04:57:57.036 INFO:tasks.workunit.client.0.smithi152.stderr:rm: cannot remove '/tmp/rbd_import_export_85090/sparse3': No such file or directory 2024-04-26T04:57:57.036 INFO:tasks.workunit.client.0.smithi152.stderr:+ true 2024-04-26T04:57:57.037 INFO:tasks.workunit.client.0.smithi152.stdout:OK 2024-04-26T04:57:57.038 INFO:tasks.workunit.client.0.smithi152.stderr:+ echo OK 2024-04-26T04:57:57.038 INFO:tasks.workunit.client.0.smithi152.stderr:+ rm -rf /tmp/rbd_import_export_85090 2024-04-26T04:57:57.038 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-04-26T04:57:57.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-04-26T04:57:57.105 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-04-26T04:57:57.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-04-26T04:57:57.365 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:57 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3781144141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:57.365 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:57 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3781144141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:57.366 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:57.367 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:57:57.391 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:57:57.391 DEBUG:teuthology.orchestra.run.smithi152:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:57:57.445 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:57:57.445 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-26T04:57:57.445 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-04-26T04:57:57.446 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:57.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:57 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3781144141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:58.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:57:58 smithi152 ceph-mon[27560]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.1 MiB/s wr, 636 op/s 2024-04-26T04:57:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:57:58 smithi152 ceph-mon[39324]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.1 MiB/s wr, 636 op/s 2024-04-26T04:57:58.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:57:58 smithi195 ceph-mon[31427]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.1 MiB/s wr, 636 op/s 2024-04-26T04:58:00.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:00 smithi152 ceph-mon[39324]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 427 op/s 2024-04-26T04:58:00.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:00 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2854503039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:00.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:00 smithi152 ceph-mon[27560]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 427 op/s 2024-04-26T04:58:00.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:00 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2854503039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:00.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:00 smithi195 ceph-mon[31427]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 427 op/s 2024-04-26T04:58:00.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:00 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2854503039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:01.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:58:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:02 smithi152 ceph-mon[39324]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:58:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:02 smithi152 ceph-mon[39324]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 368 op/s 2024-04-26T04:58:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:02 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2151770680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:02.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:02 smithi152 ceph-mon[27560]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:58:02.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:02 smithi152 ceph-mon[27560]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 368 op/s 2024-04-26T04:58:02.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:02 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2151770680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:02.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:02] "GET /metrics HTTP/1.1" 200 239815 "" "Prometheus/2.43.0" 2024-04-26T04:58:02.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:02 smithi195 ceph-mon[31427]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:58:02.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:02 smithi195 ceph-mon[31427]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 368 op/s 2024-04-26T04:58:02.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:02 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2151770680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:04.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:04 smithi152 ceph-mon[27560]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-04-26T04:58:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:04 smithi152 ceph-mon[39324]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-04-26T04:58:04.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:04 smithi195 ceph-mon[31427]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-04-26T04:58:05.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:05 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/125717691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:05.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:05 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/125717691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:05 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/125717691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:06 smithi152 ceph-mon[27560]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 879 KiB/s rd, 1.2 MiB/s wr, 285 op/s 2024-04-26T04:58:06.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:06 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:06.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:06 smithi152 ceph-mon[39324]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 879 KiB/s rd, 1.2 MiB/s wr, 285 op/s 2024-04-26T04:58:06.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:06 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:06 smithi195 ceph-mon[31427]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 879 KiB/s rd, 1.2 MiB/s wr, 285 op/s 2024-04-26T04:58:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:06 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:07.748 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:07 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3953438057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:07.748 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:07 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3953438057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:07.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:07 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3953438057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:08 smithi195 ceph-mon[31427]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:08.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:08 smithi152 ceph-mon[27560]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:08.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:08 smithi152 ceph-mon[39324]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:09.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:09 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3986396882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:09.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:09 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3986396882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:09.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:09 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3986396882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:10.681 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:10 smithi195 ceph-mon[31427]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:10.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:10 smithi152 ceph-mon[27560]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:10.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:10 smithi152 ceph-mon[39324]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:11.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:58:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:12 smithi152 ceph-mon[27560]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:12 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1029137648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:12.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:12 smithi152 ceph-mon[39324]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:12.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:12 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1029137648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:12.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:12 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:12] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2024-04-26T04:58:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:12 smithi195 ceph-mon[31427]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:12 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1029137648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:14 smithi195 ceph-mon[31427]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:14 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1035097492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:14 smithi152 ceph-mon[39324]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:14 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1035097492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:14.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:14 smithi152 ceph-mon[27560]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:14.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:14 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1035097492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:16.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:16 smithi195 ceph-mon[31427]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:16.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:16 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/121775138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:16.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:16 smithi152 ceph-mon[27560]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:16.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:16 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/121775138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:16.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:16 smithi152 ceph-mon[39324]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:16.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:16 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/121775138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:18 smithi195 ceph-mon[31427]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:18 smithi152 ceph-mon[27560]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:18.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:18 smithi152 ceph-mon[39324]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:19 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1571536184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:19.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:19 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1571536184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:19.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:19 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1571536184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:20.683 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:20 smithi195 ceph-mon[31427]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:20.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:20 smithi152 ceph-mon[27560]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:20.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:20 smithi152 ceph-mon[39324]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:21.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[33517]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:58:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:21 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/453737549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:21 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:21.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:21 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/453737549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:21.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:21 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:21.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:21 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/453737549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:21.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:21 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:22.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:22 smithi152 ceph-mon[27560]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:22.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:22 smithi152 ceph-mon[39324]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:22.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:22 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:22] "GET /metrics HTTP/1.1" 200 239857 "" "Prometheus/2.43.0" 2024-04-26T04:58:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:22 smithi195 ceph-mon[31427]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:24 smithi195 ceph-mon[31427]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:24 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/4276981136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:24 smithi152 ceph-mon[27560]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:24 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4276981136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:24 smithi152 ceph-mon[39324]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:24 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/4276981136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:26.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:26 smithi195 ceph-mon[31427]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:26.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:26 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1449598505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:26.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:26 smithi152 ceph-mon[27560]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:26.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:26 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1449598505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:26.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:26 smithi152 ceph-mon[39324]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:26.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:26 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1449598505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:28.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:28 smithi195 ceph-mon[31427]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:28.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:28 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/829930072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:28.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:28 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:28.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:28 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:28.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:28 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:28.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:28 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:28.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:28 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[27560]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/829930072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[39324]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/829930072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:28.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:28.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:28.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:28.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:28.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:28 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:29.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:29 smithi195 ceph-mon[31427]: Upgrade: Updating mgr.x 2024-04-26T04:58:29.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:29 smithi195 ceph-mon[31427]: Deploying daemon mgr.x on smithi195 2024-04-26T04:58:29.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:29 smithi152 ceph-mon[27560]: Upgrade: Updating mgr.x 2024-04-26T04:58:29.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:29 smithi152 ceph-mon[27560]: Deploying daemon mgr.x on smithi195 2024-04-26T04:58:29.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:29 smithi152 ceph-mon[39324]: Upgrade: Updating mgr.x 2024-04-26T04:58:29.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:29 smithi152 ceph-mon[39324]: Deploying daemon mgr.x on smithi195 2024-04-26T04:58:30.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:30 smithi195 systemd[1]: Stopping Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:30.602 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:30 smithi195 podman[66443]: 2024-04-26 04:58:30.304639156 +0000 UTC m=+0.145937341 container died 0f1dfdd9436ffd0eff02c4360d289369889fdaa7eeb643daa15e6b194f5651fb (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:58:30.602 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:30 smithi195 ceph-mon[31427]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:30.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:30 smithi152 ceph-mon[27560]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:30.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:30 smithi152 ceph-mon[39324]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:31.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:30 smithi195 podman[66443]: 2024-04-26 04:58:30.756226091 +0000 UTC m=+0.597524317 container cleanup 0f1dfdd9436ffd0eff02c4360d289369889fdaa7eeb643daa15e6b194f5651fb (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-26T04:58:31.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:30 smithi195 bash[66443]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:58:31.391 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 podman[66462]: 2024-04-26 04:58:31.04936386 +0000 UTC m=+0.737345169 container remove 0f1dfdd9436ffd0eff02c4360d289369889fdaa7eeb643daa15e6b194f5651fb (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:58:31.391 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:58:31.391 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:58:31.391 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 systemd[1]: Stopped Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:31.392 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service: Consumed 17.521s CPU time. 2024-04-26T04:58:31.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:31 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3830696964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:31.643 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 systemd[1]: Starting Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:31.643 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 podman[66597]: 2024-04-26 04:58:31.510662642 +0000 UTC m=+0.018045486 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:31.643 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 podman[66597]: 2024-04-26 04:58:31.642555388 +0000 UTC m=+0.149938220 container create e4b994576e20711df72c338c6ac6023e296b5adeecd5963adfd99e5326440160 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T04:58:31.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:31 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3830696964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:31.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:31 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3830696964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:31.908 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 podman[66597]: 2024-04-26 04:58:31.878677944 +0000 UTC m=+0.386060775 container init e4b994576e20711df72c338c6ac6023e296b5adeecd5963adfd99e5326440160 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-26T04:58:31.908 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 podman[66597]: 2024-04-26 04:58:31.883959103 +0000 UTC m=+0.391341935 container start e4b994576e20711df72c338c6ac6023e296b5adeecd5963adfd99e5326440160 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T04:58:32.229 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 ceph-mgr[66635]: -- 172.21.15.195:0/917987695 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557dd80ac5a0 con 0x557dd8080800 2024-04-26T04:58:32.229 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 bash[66597]: e4b994576e20711df72c338c6ac6023e296b5adeecd5963adfd99e5326440160 2024-04-26T04:58:32.229 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:31 smithi195 systemd[1]: Started Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:32.229 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:32.034+0000 7f7885b22200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:58:32.229 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:32.227+0000 7f7885b22200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:32.530 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:32.341+0000 7f7885b22200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:58:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-mon[31427]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:32.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:32] "GET /metrics HTTP/1.1" 200 239857 "" "Prometheus/2.43.0" 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[27560]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[39324]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:32 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:33.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:32.672+0000 7f7885b22200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:58:33.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:32.762+0000 7f7885b22200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:58:33.755 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:33 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2150076163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:33.755 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:33 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2150076163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:33.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:33 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2150076163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:34.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:33 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:33.982+0000 7f7885b22200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:58:34.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:34 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:34.156+0000 7f7885b22200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:34.622 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:34 smithi195 ceph-mon[31427]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:34.623 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:34 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:34.331+0000 7f7885b22200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:58:34.718 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:34 smithi152 ceph-mon[27560]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:34.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:34 smithi152 ceph-mon[39324]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:34.924 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:34 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:34.621+0000 7f7885b22200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:58:34.924 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:34 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:34.715+0000 7f7885b22200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:58:35.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:34 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:34.922+0000 7f7885b22200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:58:35.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:35 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:35.119+0000 7f7885b22200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:36.004 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:35 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:35.738+0000 7f7885b22200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:58:36.275 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:36.168+0000 7f7885b22200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:58:36.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:36.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:36.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-mon[31427]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:36.275 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3982391601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:36.295 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:36.295 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:36.295 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[39324]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:36.295 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3982391601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:36.295 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:36.295 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:36.295 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[27560]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:36.296 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:36 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3982391601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:36.530 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:36.267+0000 7f7885b22200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:58:36.999 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:36.472+0000 7f7885b22200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:58:37.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:36 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:36.998+0000 7f7885b22200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:58:37.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:37 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:37.094+0000 7f7885b22200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:58:37.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:37 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:37.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:37 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:37.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:37 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:37.708 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:37 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:37.346+0000 7f7885b22200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:58:37.708 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:37 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:37.482+0000 7f7885b22200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:58:37.708 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:37 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:37.589+0000 7f7885b22200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:58:37.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:37 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:37.707+0000 7f7885b22200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:38.017 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:37 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:37.802+0000 7f7885b22200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:58:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-mon[31427]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3716790286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:38 smithi152 ceph-mon[27560]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:38 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3716790286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:38 smithi152 ceph-mon[39324]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:38 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3716790286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:38.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:38.629+0000 7f7885b22200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:58:39.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:39 smithi195 ceph-mon[31427]: Standby manager daemon x restarted 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:39 smithi195 ceph-mon[31427]: Standby manager daemon x started 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:39 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:39 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:39 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:39 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:38.851+0000 7f7885b22200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:38] ENGINE Bus STARTING 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: CherryPy Checker: 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: The Application mounted at '' has an empty config. 2024-04-26T04:58:39.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:39.282 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:38] ENGINE Serving on http://:::9283 2024-04-26T04:58:39.282 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:38] ENGINE Bus STARTED 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[27560]: Standby manager daemon x restarted 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[27560]: Standby manager daemon x started 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[39324]: Standby manager daemon x restarted 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[39324]: Standby manager daemon x started 2024-04-26T04:58:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:58:39.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:39.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:58:39.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:39 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:40.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:40 smithi152 ceph-mon[27560]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:58:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:40 smithi152 ceph-mon[27560]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:40.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:40 smithi152 ceph-mon[39324]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:58:40.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:40 smithi152 ceph-mon[39324]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:40 smithi195 ceph-mon[31427]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:58:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:40 smithi195 ceph-mon[31427]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:40.801 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:40 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:58:41.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:41 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2507261483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:41.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:41 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2507261483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:41.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:41 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2507261483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:42.304 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:42 smithi152 ceph-mon[27560]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:42.304 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:42 smithi152 ceph-mon[39324]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:42.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:42 smithi195 ceph-mon[31427]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:42.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:42 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:42] "GET /metrics HTTP/1.1" 200 239794 "" "Prometheus/2.43.0" 2024-04-26T04:58:43.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:43 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1102345818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:43.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:43 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1102345818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:43.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:43 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1102345818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:44.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:44 smithi195 ceph-mon[31427]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:44.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:44 smithi152 ceph-mon[27560]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:44.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:44 smithi152 ceph-mon[39324]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:45.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:45 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2277321369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:45.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:45 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2277321369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:45 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2277321369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:46.473 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:46 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.473 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:46 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.473 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:46 smithi195 ceph-mon[31427]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:46.474 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:46 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.474 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:46 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[27560]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[39324]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:46 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:48.677 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:48 smithi195 ceph-mon[31427]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:48.677 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:48 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/729223302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:48.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:48 smithi152 ceph-mon[27560]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:48.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:48 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/729223302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:48 smithi152 ceph-mon[39324]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:48 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/729223302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:49.303 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:49] ENGINE Bus STOPPING 2024-04-26T04:58:49.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:49.284+0000 7fea03126700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:58:49.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setuser ceph since I am not root 2024-04-26T04:58:49.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ignoring --setgroup ceph since I am not root 2024-04-26T04:58:49.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:49 smithi152 ceph-mgr[27842]: -- 172.21.15.152:0/2130526926 <== mon.0 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55594ad5a9c0 con 0x55594ad30800 2024-04-26T04:58:49.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:49.483+0000 7fe137d4c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:58:49.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:58:49.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:49] ENGINE Bus STOPPED 2024-04-26T04:58:49.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:49] ENGINE Bus STARTING 2024-04-26T04:58:49.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e5f21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e5f21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e5f21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e5f21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e5f21700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e271a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e271a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e271a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e271a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:58:49.404+0000 7f77e271a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:49] ENGINE Serving on http://:::9283 2024-04-26T04:58:49.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:58:49] ENGINE Bus STARTED 2024-04-26T04:58:50.079 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:49.652+0000 7fe137d4c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:50.079 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:49.762+0000 7fe137d4c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:58:50.079 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:50.078+0000 7fe137d4c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:58:50.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:50.164+0000 7fe137d4c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:58:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:50.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: Activating manager daemon x 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: mgrmap e22: x(active, starting, since 0.00828633s) 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:58:50.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:58:50.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:58:50.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:58:50.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:58:50.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:58:50.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:58:50.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:58:50.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: Manager daemon x is now available 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:50.383 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:50.384 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:50.384 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:50.384 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:50.384 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: Deploying cephadm binary to smithi195 2024-04-26T04:58:50.384 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: Deploying cephadm binary to smithi152 2024-04-26T04:58:50.384 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3979336111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:58:50.385 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: Activating manager daemon x 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: mgrmap e22: x(active, starting, since 0.00828633s) 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:58:50.386 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: Manager daemon x is now available 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:50.387 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:50.388 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:50.388 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:50.388 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:50.388 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: Deploying cephadm binary to smithi195 2024-04-26T04:58:50.388 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: Deploying cephadm binary to smithi152 2024-04-26T04:58:50.388 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:50 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3979336111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:50.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:50.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' 2024-04-26T04:58:50.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:50.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 172.21.15.152:0/69868264' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:58:50.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:58:50.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: Activating manager daemon x 2024-04-26T04:58:50.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: mgrmap e22: x(active, starting, since 0.00828633s) 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:58:50.533 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:58:50.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:58:50.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:58:50.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:58:50.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:58:50.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:58:50.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:58:50.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:58:50.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:58:50.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: Manager daemon x is now available 2024-04-26T04:58:50.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:50.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:50.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: Deploying cephadm binary to smithi195 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: Deploying cephadm binary to smithi152 2024-04-26T04:58:50.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3979336111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:51.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:58:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:58:50] "GET /metrics HTTP/1.1" 200 30866 "" "Prometheus/2.43.0" 2024-04-26T04:58:51.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:51 smithi195 ceph-mon[31427]: [26/Apr/2024:04:58:50] ENGINE Bus STARTING 2024-04-26T04:58:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:51 smithi195 ceph-mon[31427]: [26/Apr/2024:04:58:50] ENGINE Serving on http://172.21.15.195:8765 2024-04-26T04:58:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:51 smithi195 ceph-mon[31427]: mgrmap e23: x(active, since 1.01027s) 2024-04-26T04:58:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:51 smithi195 ceph-mon[31427]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:51 smithi195 ceph-mon[31427]: [26/Apr/2024:04:58:50] ENGINE Serving on https://172.21.15.195:7150 2024-04-26T04:58:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:51 smithi195 ceph-mon[31427]: [26/Apr/2024:04:58:50] ENGINE Bus STARTED 2024-04-26T04:58:51.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:51.318+0000 7fe137d4c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:58:51.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:51.485+0000 7fe137d4c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[27560]: [26/Apr/2024:04:58:50] ENGINE Bus STARTING 2024-04-26T04:58:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[27560]: [26/Apr/2024:04:58:50] ENGINE Serving on http://172.21.15.195:8765 2024-04-26T04:58:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[27560]: mgrmap e23: x(active, since 1.01027s) 2024-04-26T04:58:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[27560]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[27560]: [26/Apr/2024:04:58:50] ENGINE Serving on https://172.21.15.195:7150 2024-04-26T04:58:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[27560]: [26/Apr/2024:04:58:50] ENGINE Bus STARTED 2024-04-26T04:58:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[39324]: [26/Apr/2024:04:58:50] ENGINE Bus STARTING 2024-04-26T04:58:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[39324]: [26/Apr/2024:04:58:50] ENGINE Serving on http://172.21.15.195:8765 2024-04-26T04:58:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[39324]: mgrmap e23: x(active, since 1.01027s) 2024-04-26T04:58:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[39324]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[39324]: [26/Apr/2024:04:58:50] ENGINE Serving on https://172.21.15.195:7150 2024-04-26T04:58:51.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-mon[39324]: [26/Apr/2024:04:58:50] ENGINE Bus STARTED 2024-04-26T04:58:51.921 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:51.654+0000 7fe137d4c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:58:52.297 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:51.920+0000 7fe137d4c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:58:52.297 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:52.015+0000 7fe137d4c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:58:52.297 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:52.113+0000 7fe137d4c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:58:52.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-mon[27560]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:52.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-mon[27560]: mgrmap e24: x(active, since 2s) 2024-04-26T04:58:52.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3279906574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-mon[39324]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-mon[39324]: mgrmap e24: x(active, since 2s) 2024-04-26T04:58:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3279906574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:52.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:52.401+0000 7fe137d4c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:52.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:52 smithi195 ceph-mon[31427]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:52.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:52 smithi195 ceph-mon[31427]: mgrmap e24: x(active, since 2s) 2024-04-26T04:58:52.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:52 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3279906574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:53.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:52 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:52.866+0000 7fe137d4c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:58:53.684 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:53.406+0000 7fe137d4c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:58:53.684 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:53.502+0000 7fe137d4c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:58:53.684 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:53.683+0000 7fe137d4c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:58:54.241 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:53.978+0000 7fe137d4c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:58:54.242 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:54.240+0000 7fe137d4c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:58:54.498 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-mon[27560]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:54.499 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-mon[27560]: mgrmap e25: x(active, since 4s) 2024-04-26T04:58:54.499 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-mon[39324]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:54.499 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-mon[39324]: mgrmap e25: x(active, since 4s) 2024-04-26T04:58:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:54 smithi195 ceph-mon[31427]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:54.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:54 smithi195 ceph-mon[31427]: mgrmap e25: x(active, since 4s) 2024-04-26T04:58:54.863 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:54.497+0000 7fe137d4c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:58:54.864 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:54.639+0000 7fe137d4c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:58:54.864 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:54.739+0000 7fe137d4c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:58:54.864 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:54.862+0000 7fe137d4c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:55.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:54.964+0000 7fe137d4c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:58:55.493 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3784284656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:55.629 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3784284656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3784284656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:56.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:55 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:55.805+0000 7fe137d4c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:58:56.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:56.029+0000 7fe137d4c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:58:56.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:58:56] ENGINE Bus STARTING 2024-04-26T04:58:56.462 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[27560]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:56.462 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[27560]: Standby manager daemon y started 2024-04-26T04:58:56.462 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:58:56.462 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:56.462 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: CherryPy Checker: 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: The Application mounted at '' has an empty config. 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:58:56] ENGINE Serving on http://:::9283 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: [26/Apr/2024:04:58:56] ENGINE Bus STARTED 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[39324]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:56.463 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[39324]: Standby manager daemon y started 2024-04-26T04:58:56.464 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:58:56.464 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:56.464 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:58:56.464 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:56 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:56 smithi195 ceph-mon[31427]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:56 smithi195 ceph-mon[31427]: Standby manager daemon y started 2024-04-26T04:58:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:56 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:58:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:56 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:56 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:58:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:56 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1738109911' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:57.457 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:57 smithi152 ceph-mon[27560]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:58:57.457 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:57 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:58:57.457 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:57 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/169874096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:57.457 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:57 smithi152 ceph-mon[39324]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:58:57.457 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:57 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:58:57.457 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:57 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/169874096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:57.600 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:57 smithi195 ceph-mon[31427]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:58:57.600 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:57 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:58:57.600 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:57 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/169874096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:59.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:58 smithi195 ceph-mon[31427]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:58:59.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:58 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:59.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:58 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:59.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:58 smithi152 ceph-mon[27560]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:58:59.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:58 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:59.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:58 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:58 smithi152 ceph-mon[39324]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:58:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:58 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:58:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:58 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi152 to 2480M 2024-04-26T04:59:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:58:59 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1931939619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi152 to 2480M 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1931939619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:00.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi152 to 2480M 2024-04-26T04:59:00.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:00.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:58:59 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1931939619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:00.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:59:00] "GET /metrics HTTP/1.1" 200 30866 "" "Prometheus/2.43.0" 2024-04-26T04:59:01.187 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:00 smithi195 ceph-mon[31427]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:01.187 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:00 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:01.188 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:00 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:00 smithi152 ceph-mon[27560]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:00 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:01.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:00 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:00 smithi152 ceph-mon[39324]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:00 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:01.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:00 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:02.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:01 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2835751316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:02.302 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:01 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2835751316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:02.302 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:01 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2835751316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:02.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:59:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:59:03.021 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:02 smithi195 ceph-mon[31427]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:59:03.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:02 smithi152 ceph-mon[27560]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:59:03.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:02 smithi152 ceph-mon[39324]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:59:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: Adjusting osd_memory_target on smithi195 to 2480M 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:59:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3217559200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:04.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: Adjusting osd_memory_target on smithi195 to 2480M 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:59:04.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:04 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3217559200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: Adjusting osd_memory_target on smithi195 to 2480M 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:04.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:04.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:04.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:04 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3217559200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:05.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:05.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:05.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:05.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[27560]: Reconfiguring daemon alertmanager.a on smithi152 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:59:05.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:05 smithi152 ceph-mon[39324]: Reconfiguring daemon alertmanager.a on smithi152 2024-04-26T04:59:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:59:05.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:05 smithi195 ceph-mon[31427]: Reconfiguring daemon alertmanager.a on smithi152 2024-04-26T04:59:06.715 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:06 smithi152 ceph-mon[39324]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:06.715 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:06 smithi152 ceph-mon[27560]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:06 smithi195 ceph-mon[31427]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:07.486 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:07 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1905769482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:07.486 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:07 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1905769482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:07.741 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:07 smithi152 systemd[1]: Stopping Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:07.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:07 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1905769482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:07.998 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[76426]: ts=2024-04-26T04:59:07.892Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:59:07.998 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:07 smithi152 podman[91360]: 2024-04-26 04:59:07.923289858 +0000 UTC m=+0.164910063 container died 92be36185dbe12bd7a282534332ca68385755ec0e04bc35705fff08b6e6a3c07 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:08.575 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:08 smithi152 ceph-mon[27560]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:08.575 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:08 smithi152 ceph-mon[39324]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:08.576 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 podman[91360]: 2024-04-26 04:59:08.307133592 +0000 UTC m=+0.548753803 container cleanup 92be36185dbe12bd7a282534332ca68385755ec0e04bc35705fff08b6e6a3c07 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:08.576 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 bash[91360]: ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:59:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:08 smithi195 ceph-mon[31427]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:59:08.826 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 podman[91379]: 2024-04-26 04:59:08.575383665 +0000 UTC m=+0.647895235 container remove 92be36185dbe12bd7a282534332ca68385755ec0e04bc35705fff08b6e6a3c07 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:08.826 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 podman[91379]: 2024-04-26 04:59:08.659420253 +0000 UTC m=+0.731931821 volume remove 71b2c167b01e88e58a0c1012018b95eab3d4011a2d4440397b32e581a2c4f46c 2024-04-26T04:59:08.826 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:59:08.826 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 systemd[1]: Stopped Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:08.827 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.180s CPU time. 2024-04-26T04:59:08.827 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:08 smithi152 systemd[1]: Starting Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:09.339 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 podman[91507]: 2024-04-26 04:59:08.964120308 +0000 UTC m=+0.018037908 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:59:09.339 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 podman[91507]: 2024-04-26 04:59:09.085701977 +0000 UTC m=+0.139619562 volume create 8ec13b9675f522859ba8f96f8925264ede50a0f2807ba9f217850ced27ca101e 2024-04-26T04:59:09.339 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 podman[91507]: 2024-04-26 04:59:09.161080088 +0000 UTC m=+0.214997669 container create 252d0e292dd9f588a86941640a745a9677ff93dde8e180486d064a8f539d92d7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:09.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:09 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/47410459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:09.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/47410459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:09.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/47410459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:09.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 podman[91507]: 2024-04-26 04:59:09.430540232 +0000 UTC m=+0.484457815 container init 252d0e292dd9f588a86941640a745a9677ff93dde8e180486d064a8f539d92d7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:09.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 podman[91507]: 2024-04-26 04:59:09.435960075 +0000 UTC m=+0.489877658 container start 252d0e292dd9f588a86941640a745a9677ff93dde8e180486d064a8f539d92d7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:09.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.447Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:59:09.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.447Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:59:09.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.448Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.152 port=9094 2024-04-26T04:59:09.881 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.449Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:59:09.881 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.472Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:59:09.881 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.472Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:59:09.881 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.475Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:59:09.881 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:09.475Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:59:09.881 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 bash[91507]: 252d0e292dd9f588a86941640a745a9677ff93dde8e180486d064a8f539d92d7 2024-04-26T04:59:09.881 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:09 smithi152 systemd[1]: Started Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:10.380 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:10 smithi152 systemd[1]: Stopping Ceph node-exporter.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:10.685 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:10 smithi152 podman[91680]: 2024-04-26 04:59:10.39706547 +0000 UTC m=+0.149375706 container died 84ac3154b2e4618cad9fbd1c9dcb71a13021fa31336dfa916bb987d9e5b6d4aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:10.685 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[27560]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:10.685 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:10.685 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:10.685 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[27560]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:59:10.686 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[27560]: Reconfiguring daemon node-exporter.a on smithi152 2024-04-26T04:59:10.686 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[39324]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:10.686 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:10.686 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:10.686 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[39324]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:59:10.686 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:10 smithi152 ceph-mon[39324]: Reconfiguring daemon node-exporter.a on smithi152 2024-04-26T04:59:11.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:59:10] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.43.0" 2024-04-26T04:59:11.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:10 smithi195 ceph-mon[31427]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:11.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:10 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:11.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:10 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:11.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:10 smithi195 ceph-mon[31427]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:59:11.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:10 smithi195 ceph-mon[31427]: Reconfiguring daemon node-exporter.a on smithi152 2024-04-26T04:59:11.060 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:10 smithi152 podman[91680]: 2024-04-26 04:59:10.792610873 +0000 UTC m=+0.544921096 container cleanup 84ac3154b2e4618cad9fbd1c9dcb71a13021fa31336dfa916bb987d9e5b6d4aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:11.060 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:10 smithi152 bash[91680]: ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T04:59:11.380 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 podman[91700]: 2024-04-26 04:59:11.060523795 +0000 UTC m=+0.666572195 container remove 84ac3154b2e4618cad9fbd1c9dcb71a13021fa31336dfa916bb987d9e5b6d4aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:11.380 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:59:11.380 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:11.380 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 systemd[1]: Stopped Ceph node-exporter.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:11.380 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.229s CPU time. 2024-04-26T04:59:11.380 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 systemd[1]: Starting Ceph node-exporter.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:11.756 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:11.450Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000996927s 2024-04-26T04:59:11.756 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 podman[91818]: 2024-04-26 04:59:11.340911491 +0000 UTC m=+0.017495542 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:59:11.756 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 podman[91818]: 2024-04-26 04:59:11.470140474 +0000 UTC m=+0.146724510 container create 8d4a4fd1dbaef8625c7daddeab48d0530c02a7b8eb6cc32cfb325df8740bcc4e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:11.756 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 podman[91818]: 2024-04-26 04:59:11.672679821 +0000 UTC m=+0.349263857 container init 8d4a4fd1dbaef8625c7daddeab48d0530c02a7b8eb6cc32cfb325df8740bcc4e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:11.756 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 podman[91818]: 2024-04-26 04:59:11.677984125 +0000 UTC m=+0.354568162 container start 8d4a4fd1dbaef8625c7daddeab48d0530c02a7b8eb6cc32cfb325df8740bcc4e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:11.756 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.681Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.681Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.681Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.681Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:59:11.757 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:59:11.758 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:59:11.759 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:59:11.760 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:59:11.761 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:59:11.761 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:59:11.761 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:59:11.761 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a[91832]: ts=2024-04-26T04:59:11.682Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:59:11.761 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 bash[91818]: 8d4a4fd1dbaef8625c7daddeab48d0530c02a7b8eb6cc32cfb325df8740bcc4e 2024-04-26T04:59:12.111 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 04:59:11 smithi152 systemd[1]: Started Ceph node-exporter.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:12.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:12 smithi195 ceph-mon[31427]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:12 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/735780694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:12 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:12 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:12 smithi195 ceph-mon[31427]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:59:12.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[27819]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:59:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:59:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[27560]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/735780694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[27560]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:59:12.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[39324]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:12.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/735780694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:12.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:12.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:12.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:12 smithi152 ceph-mon[39324]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:59:13.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:13 smithi195 ceph-mon[31427]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:59:13.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:13 smithi152 ceph-mon[27560]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:59:13.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:13 smithi152 ceph-mon[39324]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:59:14.261 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:14 smithi195 ceph-mon[31427]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:14.261 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:14 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/154689746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:14.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:14 smithi152 ceph-mon[27560]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:14.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:14 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/154689746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:14.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:14 smithi152 ceph-mon[39324]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:14.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:14 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/154689746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:15.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 systemd[1]: Stopping Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:16.188 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.889Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.890Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.890Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.890Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:59:16.189 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[63532]: ts=2024-04-26T04:59:15.890Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:59:16.190 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:15 smithi195 podman[75382]: 2024-04-26 04:59:15.899994492 +0000 UTC m=+0.126214181 container died a731aeae0e891d1d14070bfd4344f27ab80c8356f42f5ed9ab81fb7fc4ab001b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:16 smithi195 ceph-mon[31427]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:16.531 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:16 smithi195 podman[75382]: 2024-04-26 04:59:16.387567262 +0000 UTC m=+0.613786950 container cleanup a731aeae0e891d1d14070bfd4344f27ab80c8356f42f5ed9ab81fb7fc4ab001b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:16.531 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:16 smithi195 bash[75382]: ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:59:16.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:16 smithi152 ceph-mon[27560]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:16.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:16 smithi152 ceph-mon[39324]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:17.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:16 smithi195 podman[75401]: 2024-04-26 04:59:16.647471335 +0000 UTC m=+0.750267845 container remove a731aeae0e891d1d14070bfd4344f27ab80c8356f42f5ed9ab81fb7fc4ab001b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:17.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:16 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:59:17.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:16 smithi195 systemd[1]: Stopped Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:17.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:16 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.717s CPU time. 2024-04-26T04:59:17.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:16 smithi195 systemd[1]: Starting Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:17.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 podman[75504]: 2024-04-26 04:59:16.935567217 +0000 UTC m=+0.019002561 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:59:17.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 podman[75504]: 2024-04-26 04:59:17.031700161 +0000 UTC m=+0.115135551 container create 0bf2676a7051416d2b564317b156fc7f5d21fc3d4945cb5a57751560a92275fc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:17.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 podman[75504]: 2024-04-26 04:59:17.250584906 +0000 UTC m=+0.334020250 container init 0bf2676a7051416d2b564317b156fc7f5d21fc3d4945cb5a57751560a92275fc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:17.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 podman[75504]: 2024-04-26 04:59:17.255876237 +0000 UTC m=+0.339311584 container start 0bf2676a7051416d2b564317b156fc7f5d21fc3d4945cb5a57751560a92275fc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:17.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.279Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.279Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.279Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi195 (none))" 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.279Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.279Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.280Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.281Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.281Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:59:17.283 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.281Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:59:17.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:17 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1134218020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:17.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:17 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1134218020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1134218020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.284Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.284Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.043µs 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.284Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.284Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.298Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.298Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.298Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.762µs wal_replay_duration=14.412192ms wbl_replay_duration=124ns total_replay_duration=14.450486ms 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.301Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.301Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:59:17.781 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.301Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:59:17.782 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.321Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.954454ms db_storage=806ns remote_storage=1.052µs web_handler=455ns query_engine=552ns scrape=401.575µs scrape_sd=96.968µs notify=18.915µs notify_sd=24.672µs rules=19.197156ms tracing=6.188µs 2024-04-26T04:59:17.782 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.321Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:59:17.782 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:17.321Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:59:17.782 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 bash[75504]: 0bf2676a7051416d2b564317b156fc7f5d21fc3d4945cb5a57751560a92275fc 2024-04-26T04:59:17.782 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:17 smithi195 systemd[1]: Started Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:18 smithi195 ceph-mon[31427]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:18 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:18 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:18 smithi195 ceph-mon[31427]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:59:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:18 smithi195 ceph-mon[31427]: Reconfiguring daemon grafana.a on smithi195 2024-04-26T04:59:18.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[27560]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[27560]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[27560]: Reconfiguring daemon grafana.a on smithi195 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[39324]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[39324]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:59:18.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:18 smithi152 ceph-mon[39324]: Reconfiguring daemon grafana.a on smithi195 2024-04-26T04:59:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:19 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2895951267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:19 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:19 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2895951267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:19.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:19 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:19.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:19 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2895951267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:19.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:19 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:19.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 04:59:19 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T04:59:19.453Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004036947s 2024-04-26T04:59:20.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:20 smithi195 ceph-mon[31427]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:20.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:20 smithi152 ceph-mon[27560]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:20.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:20 smithi152 ceph-mon[39324]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:21 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1659062674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:21.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:21 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1659062674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:21.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:21 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1659062674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:22 smithi195 ceph-mon[31427]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:22.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:22 smithi152 ceph-mon[27560]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:22.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:22 smithi152 ceph-mon[39324]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:24 smithi195 ceph-mon[31427]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:24.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:24 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/37096202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:24 smithi152 ceph-mon[27560]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:24 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/37096202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:24 smithi152 ceph-mon[39324]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:24 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/37096202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:26.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:26 smithi195 ceph-mon[31427]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:26.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:26 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1031091467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:26.879 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:26 smithi152 ceph-mon[39324]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:26.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:26 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1031091467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:26.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:26 smithi152 ceph-mon[27560]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:26.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:26 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1031091467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:26.962 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:26 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:59:26] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.43.0" 2024-04-26T04:59:27.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:27 smithi195 systemd[1]: Stopping Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:27.582 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:27 smithi195 podman[75901]: 2024-04-26 04:59:27.286408769 +0000 UTC m=+0.122150307 container died 8dd84a1289295be629fbdc186255a3046b51f4c9ffd6c433a078d24ec90b3e1f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, version=8.5, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664) 2024-04-26T04:59:28.000 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:27 smithi195 podman[75901]: 2024-04-26 04:59:27.740216983 +0000 UTC m=+0.575958520 container cleanup 8dd84a1289295be629fbdc186255a3046b51f4c9ffd6c433a078d24ec90b3e1f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, description=Ceph Grafana Container, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:59:28.000 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:27 smithi195 bash[75901]: ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a 2024-04-26T04:59:28.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 podman[75920]: 2024-04-26 04:59:27.999692529 +0000 UTC m=+0.716111322 container remove 8dd84a1289295be629fbdc186255a3046b51f4c9ffd6c433a078d24ec90b3e1f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, vcs-type=git) 2024-04-26T04:59:28.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:59:28.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:28.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:28.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.284s CPU time. 2024-04-26T04:59:28.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:28.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:28 smithi195 ceph-mon[31427]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:28.654 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:28 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3394667675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:28.654 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 podman[76023]: 2024-04-26 04:59:28.278536063 +0000 UTC m=+0.017249075 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:28.654 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 podman[76023]: 2024-04-26 04:59:28.40928558 +0000 UTC m=+0.147998589 container create fe3aaa4146989ce03ea203cd0048fe385c22a5ca050a5d445e7a2897285f28df (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:59:28.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:28 smithi152 ceph-mon[39324]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:28.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:28 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3394667675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:28 smithi152 ceph-mon[27560]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:28 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3394667675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:29.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:29.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 podman[76023]: 2024-04-26 04:59:28.653349007 +0000 UTC m=+0.392062016 container init fe3aaa4146989ce03ea203cd0048fe385c22a5ca050a5d445e7a2897285f28df (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, vcs-type=git, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.openshift.expose-services=, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5) 2024-04-26T04:59:29.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 podman[76023]: 2024-04-26 04:59:28.659128152 +0000 UTC m=+0.397841162 container start fe3aaa4146989ce03ea203cd0048fe385c22a5ca050a5d445e7a2897285f28df (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, description=Ceph Grafana Container, version=8.5, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, distribution-scope=public, release=239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, vendor=Red Hat, Inc.) 2024-04-26T04:59:29.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 bash[76023]: fe3aaa4146989ce03ea203cd0048fe385c22a5ca050a5d445e7a2897285f28df 2024-04-26T04:59:29.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:29.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[76038]: logger=settings t=2024-04-26T04:59:28.811099739Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:29.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:28 smithi195 podman[76074]: 2024-04-26 04:59:28.838432357 +0000 UTC m=+0.019724348 container died fe3aaa4146989ce03ea203cd0048fe385c22a5ca050a5d445e7a2897285f28df (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.buildah.version=1.32.0, version=8.5, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, maintainer=Paul Cuzner ) 2024-04-26T04:59:29.999 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:29 smithi195 systemd[1]: Stopping Ceph node-exporter.b for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:29.999 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:29 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:29.999 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:29 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:29.999 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:29 smithi195 ceph-mon[31427]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:59:29.999 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:29 smithi195 ceph-mon[31427]: Reconfiguring daemon node-exporter.b on smithi195 2024-04-26T04:59:30.000 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:29 smithi195 podman[76074]: 2024-04-26 04:59:29.741280442 +0000 UTC m=+0.922572447 container remove fe3aaa4146989ce03ea203cd0048fe385c22a5ca050a5d445e7a2897285f28df (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, distribution-scope=public, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, vendor=Red Hat, Inc., vcs-type=git) 2024-04-26T04:59:30.000 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:29 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:30.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:30.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:30.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[39324]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:59:30.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[39324]: Reconfiguring daemon node-exporter.b on smithi195 2024-04-26T04:59:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:30.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[27560]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:59:30.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:29 smithi152 ceph-mon[27560]: Reconfiguring daemon node-exporter.b on smithi195 2024-04-26T04:59:30.280 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 podman[76241]: 2024-04-26 04:59:30.124395669 +0000 UTC m=+0.111578960 container died b3a8846a9680afff27758d47e21d748edd643d318659f3a23f15c0f891f84c1e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:30.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:30 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:30.737 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 podman[76241]: 2024-04-26 04:59:30.527727296 +0000 UTC m=+0.514910595 container cleanup b3a8846a9680afff27758d47e21d748edd643d318659f3a23f15c0f891f84c1e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:30.737 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 bash[76241]: ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T04:59:31.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:30 smithi195 ceph-mon[31427]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:31.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 podman[76260]: 2024-04-26 04:59:30.737429854 +0000 UTC m=+0.617512669 container remove b3a8846a9680afff27758d47e21d748edd643d318659f3a23f15c0f891f84c1e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:31.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:59:31.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T04:59:31.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 systemd[1]: Stopped Ceph node-exporter.b for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:31.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.149s CPU time. 2024-04-26T04:59:31.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:30 smithi195 systemd[1]: Starting Ceph node-exporter.b for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:31.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:30 smithi152 ceph-mon[27560]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:31.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:30 smithi152 ceph-mon[39324]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:31.492 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 podman[76365]: 2024-04-26 04:59:31.065489107 +0000 UTC m=+0.017581931 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:59:31.492 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 podman[76365]: 2024-04-26 04:59:31.163855867 +0000 UTC m=+0.115948719 container create bdec586e73b34435bc45efb2253133bb2c1ed589d57c45251dbc1ca29ffcc5b0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:31.776 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STOPPING 2024-04-26T04:59:31.776 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:59:31.776 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STOPPED 2024-04-26T04:59:31.776 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STARTING 2024-04-26T04:59:31.776 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Serving on http://:::9283 2024-04-26T04:59:31.777 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STARTED 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 podman[76365]: 2024-04-26 04:59:31.491882416 +0000 UTC m=+0.443975228 container init bdec586e73b34435bc45efb2253133bb2c1ed589d57c45251dbc1ca29ffcc5b0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 podman[76365]: 2024-04-26 04:59:31.497574807 +0000 UTC m=+0.449667618 container start bdec586e73b34435bc45efb2253133bb2c1ed589d57c45251dbc1ca29ffcc5b0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.500Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.500Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.500Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.500Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.500Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:59:31.777 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:59:31.778 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:59:31.779 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:59:31.780 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.501Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.502Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b[76379]: ts=2024-04-26T04:59:31.502Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 bash[76365]: bdec586e73b34435bc45efb2253133bb2c1ed589d57c45251dbc1ca29ffcc5b0 2024-04-26T04:59:31.781 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 04:59:31 smithi195 systemd[1]: Started Ceph node-exporter.b for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:32.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2699883274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195:9095"}]: dispatch 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195:3000"}]: dispatch 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STOPPING 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STOPPED 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STARTING 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Serving on http://:::9283 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STARTED 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STOPPING 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STOPPED 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:31] ENGINE Bus STARTING 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:32] ENGINE Serving on http://:::9283 2024-04-26T04:59:32.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:32] ENGINE Bus STARTED 2024-04-26T04:59:32.129 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2699883274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195:9095"}]: dispatch 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195:3000"}]: dispatch 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2699883274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195:9095"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195:3000"}]: dispatch 2024-04-26T04:59:32.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:32.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:31 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195:9095"}]: dispatch 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195:3000"}]: dispatch 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:59:32.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[27560]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195:9095"}]: dispatch 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195:3000"}]: dispatch 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:59:32.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:32 smithi152 ceph-mon[39324]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:59:33.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195:9095"}]: dispatch 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195:3000"}]: dispatch 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:59:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:32 smithi195 ceph-mon[31427]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:59:34.108 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/760435780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:34.108 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:34.108 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/760435780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:59:34.109 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:33 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:33 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/760435780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:33 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:33 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:59:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:33 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:59:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:33 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:59:34.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:33 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:34.962 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[27560]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:34.963 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[27560]: Upgrade: Updating mgr.y 2024-04-26T04:59:34.963 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[27560]: Deploying daemon mgr.y on smithi152 2024-04-26T04:59:34.963 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:34.963 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[39324]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:34.963 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[39324]: Upgrade: Updating mgr.y 2024-04-26T04:59:34.964 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[39324]: Deploying daemon mgr.y on smithi152 2024-04-26T04:59:34.964 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:34 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:34 smithi195 ceph-mon[31427]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:34 smithi195 ceph-mon[31427]: Upgrade: Updating mgr.y 2024-04-26T04:59:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:34 smithi195 ceph-mon[31427]: Deploying daemon mgr.y on smithi152 2024-04-26T04:59:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:34 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:35.550 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:35 smithi152 systemd[1]: Stopping Ceph mgr.y for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:35.839 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:35 smithi152 podman[92470]: 2024-04-26 04:59:35.550206568 +0000 UTC m=+0.146426446 container died ee2b14613007676b556bcc14697c5c730d0e15fd494e75494374666b9c2bb2d5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:59:36.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:35 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1850258376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:36.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:35 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1850258376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:36.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:35 smithi152 podman[92470]: 2024-04-26 04:59:35.949967969 +0000 UTC m=+0.546187859 container cleanup ee2b14613007676b556bcc14697c5c730d0e15fd494e75494374666b9c2bb2d5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-26T04:59:36.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:35 smithi152 bash[92470]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y 2024-04-26T04:59:36.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:35 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1850258376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:36.526 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 podman[92489]: 2024-04-26 04:59:36.201515925 +0000 UTC m=+0.653293369 container remove ee2b14613007676b556bcc14697c5c730d0e15fd494e75494374666b9c2bb2d5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:59:36.526 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:59:36.527 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-26T04:59:36.527 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 systemd[1]: Stopped Ceph mgr.y for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:36.527 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service: Consumed 57.811s CPU time. 2024-04-26T04:59:36.527 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 systemd[1]: Starting Ceph mgr.y for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:36.840 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 podman[92629]: 2024-04-26 04:59:36.643615762 +0000 UTC m=+0.018057581 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:36.840 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 podman[92629]: 2024-04-26 04:59:36.753099682 +0000 UTC m=+0.127541498 container create ec2ad8342f2fccc0e4fd4cdd56b919e913b22ce973dac8affdc80c41c62dcad1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-26T04:59:37.105 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:36 smithi152 ceph-mon[39324]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:37.106 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:36 smithi152 ceph-mon[27560]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:37.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:36 smithi152 podman[92629]: 2024-04-26 04:59:36.997556605 +0000 UTC m=+0.371998414 container init ec2ad8342f2fccc0e4fd4cdd56b919e913b22ce973dac8affdc80c41c62dcad1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True) 2024-04-26T04:59:37.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 podman[92629]: 2024-04-26 04:59:37.003096406 +0000 UTC m=+0.377538216 container start ec2ad8342f2fccc0e4fd4cdd56b919e913b22ce973dac8affdc80c41c62dcad1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T04:59:37.106 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 ceph-mgr[92667]: -- 172.21.15.152:0/2231806170 <== mon.0 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c3305305a0 con 0x55c330505000 2024-04-26T04:59:37.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:36 smithi195 ceph-mon[31427]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:37.281 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:36 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:59:36] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.43.0" 2024-04-26T04:59:37.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 bash[92629]: ec2ad8342f2fccc0e4fd4cdd56b919e913b22ce973dac8affdc80c41c62dcad1 2024-04-26T04:59:37.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 systemd[1]: Started Ceph mgr.y for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:37.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:37.168+0000 7f78915d5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:59:37.860 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:37.385+0000 7f78915d5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:37.860 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:37.498+0000 7f78915d5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:59:37.860 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:37.859+0000 7f78915d5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:59:38.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:37.957+0000 7f78915d5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[27560]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/802877566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:38.435 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[39324]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:38.436 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:38 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/802877566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:38.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:38 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:38.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:38 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:38.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:38 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:38.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:38 smithi195 ceph-mon[31427]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:38.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:38 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/802877566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:39.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T04:59:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:39.628 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:39.222+0000 7f78915d5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:59:39.628 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:39.412+0000 7f78915d5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:39.628 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:39.627+0000 7f78915d5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:59:40.252 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:39.934+0000 7f78915d5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:59:40.252 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:40.029+0000 7f78915d5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:59:40.356 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-26T04:59:40.356 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:40.356 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:40.356 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 podman[76711]: 2024-04-26 04:59:40.178721199 +0000 UTC m=+0.018482017 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:40.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:40 smithi152 ceph-mon[27560]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:40 smithi152 ceph-mon[39324]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:40.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:40.251+0000 7f78915d5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:59:40.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:40 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:40.487+0000 7f78915d5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:40.662 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:40 smithi195 ceph-mon[31427]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:40.662 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 podman[76711]: 2024-04-26 04:59:40.547070893 +0000 UTC m=+0.386831699 container create 9fb4441e137169a2b6b2d0f677467828f4d98a19b0ec59120e259f4a807bdd21 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, vendor=Red Hat, Inc., description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, release=239.1651231664, architecture=x86_64, version=8.5, maintainer=Paul Cuzner ) 2024-04-26T04:59:40.964 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 podman[76711]: 2024-04-26 04:59:40.856230387 +0000 UTC m=+0.695991204 container init 9fb4441e137169a2b6b2d0f677467828f4d98a19b0ec59120e259f4a807bdd21 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, name=ubi8, io.openshift.tags=base rhel8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-type=git, version=8.5, com.redhat.component=ubi8-container, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:59:40.964 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 podman[76711]: 2024-04-26 04:59:40.862171053 +0000 UTC m=+0.701931865 container start 9fb4441e137169a2b6b2d0f677467828f4d98a19b0ec59120e259f4a807bdd21 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, version=8.5, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, release=239.1651231664, com.redhat.component=ubi8-container, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=ubi8) 2024-04-26T04:59:41.280 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 bash[76711]: 9fb4441e137169a2b6b2d0f677467828f4d98a19b0ec59120e259f4a807bdd21 2024-04-26T04:59:41.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:40 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:41.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:41 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[76769]: logger=settings t=2024-04-26T04:59:41.018015697Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:41.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:41.123+0000 7f78915d5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:59:41.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1849876640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:41.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:41.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:41.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1849876640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:41.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:41.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:41.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:41 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1849876640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:41.653 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:41 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:41.654 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:41 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:41.654 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:41 smithi195 podman[76811]: 2024-04-26 04:59:41.375578181 +0000 UTC m=+0.349658501 container died 9fb4441e137169a2b6b2d0f677467828f4d98a19b0ec59120e259f4a807bdd21 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, release=239.1651231664, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:59:41.879 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:41.576+0000 7f78915d5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:59:41.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:41.687+0000 7f78915d5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:59:42.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:42 smithi195 podman[76811]: 2024-04-26 04:59:42.012040499 +0000 UTC m=+0.986120803 container remove 9fb4441e137169a2b6b2d0f677467828f4d98a19b0ec59120e259f4a807bdd21 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, architecture=x86_64, description=Ceph Grafana Container, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public) 2024-04-26T04:59:42.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:42 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:42.358 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:41 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:41.901+0000 7f78915d5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:59:42.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-mon[27560]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:42.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:42.461+0000 7f78915d5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:59:42.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:42.578+0000 7f78915d5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:59:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-mon[39324]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:42.642 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:42 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:42.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:42 smithi195 ceph-mon[31427]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:42.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:42 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:42.642 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:42 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:43.127 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:42 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:42.860+0000 7f78915d5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:59:43.127 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:43.004+0000 7f78915d5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:59:43.127 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:43.106+0000 7f78915d5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:59:43.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:43 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2274767169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:43.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:43 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2274767169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:43.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:43.243+0000 7f78915d5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:43.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:43.344+0000 7f78915d5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:59:43.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:43 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2274767169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:44.313 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:44.186+0000 7f78915d5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:59:44.569 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-mon[27560]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:44.570 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:44.570 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:44.570 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-mon[39324]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:44.570 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:44.570 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:44.570 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:44.425+0000 7f78915d5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:59:44.571 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:44] ENGINE Bus STARTING 2024-04-26T04:59:44.571 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: CherryPy Checker: 2024-04-26T04:59:44.571 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: The Application mounted at '' has an empty config. 2024-04-26T04:59:44.571 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:44.571 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:44] ENGINE Serving on http://:::9283 2024-04-26T04:59:44.571 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:44] ENGINE Bus STARTED 2024-04-26T04:59:44.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:44 smithi195 ceph-mon[31427]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:44 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:44 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:45.515 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[27560]: Standby manager daemon y restarted 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[27560]: Standby manager daemon y started 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3196463901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[39324]: Standby manager daemon y restarted 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[39324]: Standby manager daemon y started 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:45.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:59:45.517 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:45.517 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:45 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3196463901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:45 smithi195 ceph-mon[31427]: Standby manager daemon y restarted 2024-04-26T04:59:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:45 smithi195 ceph-mon[31427]: Standby manager daemon y started 2024-04-26T04:59:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:45 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:59:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:45 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:45 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:59:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:45 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:45 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3196463901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:46.161 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:46] ENGINE Bus STOPPING 2024-04-26T04:59:46.161 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:59:46.161 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:46] ENGINE Bus STOPPED 2024-04-26T04:59:46.500 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:46.128+0000 7f7879116700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:59:46.500 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ignoring --setuser ceph since I am not root 2024-04-26T04:59:46.500 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ignoring --setgroup ceph since I am not root 2024-04-26T04:59:46.500 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mgr[66635]: -- 172.21.15.195:0/4165666296 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55861edf05a0 con 0x55861edc4800 2024-04-26T04:59:46.500 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:46.310+0000 7f081be9a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:59:46.500 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:46.498+0000 7f081be9a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: mgrmap e27: x(active, since 56s), standbys: y 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: Activating manager daemon y 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T04:59:46.501 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25240 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: mgrmap e28: y(active, starting, since 0.00983625s) 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:59:46.502 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: Manager daemon y is now available 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:46.503 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:46.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:46] ENGINE Bus STARTING 2024-04-26T04:59:46.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.261+0000 7f77ef1c0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.261+0000 7f77ef1c0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.261+0000 7f77ef1c0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.261+0000 7f77ef1c0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.261+0000 7f77ef1c0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.262+0000 7f77f21c6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.262+0000 7f77f21c6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.262+0000 7f77f21c6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.262+0000 7f77f21c6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T04:59:46.262+0000 7f77f21c6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:46] ENGINE Serving on http://:::9283 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:04:59:46] ENGINE Bus STARTED 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: mgrmap e27: x(active, since 56s), standbys: y 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:46.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: Activating manager daemon y 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25240 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: mgrmap e28: y(active, starting, since 0.00983625s) 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:59:46.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: mgrmap e27: x(active, since 56s), standbys: y 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.633 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 172.21.15.195:0/1966757043' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: Activating manager daemon y 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25240 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: mgrmap e28: y(active, starting, since 0.00983625s) 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:59:46.634 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:59:46.635 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: Manager daemon y is now available 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: Manager daemon y is now available 2024-04-26T04:59:46.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:46.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:46.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:46.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:46.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:46.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:46 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:46.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:46.608+0000 7f081be9a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:59:47.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:46 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:46.933+0000 7f081be9a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:59:47.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:47 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:47.033+0000 7f081be9a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[27560]: [26/Apr/2024:04:59:46] ENGINE Bus STARTING 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[27560]: [26/Apr/2024:04:59:46] ENGINE Serving on http://172.21.15.152:8765 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[27560]: [26/Apr/2024:04:59:46] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[27560]: [26/Apr/2024:04:59:46] ENGINE Bus STARTED 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[27560]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[27560]: mgrmap e29: y(active, since 1.01039s) 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[39324]: [26/Apr/2024:04:59:46] ENGINE Bus STARTING 2024-04-26T04:59:47.579 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[39324]: [26/Apr/2024:04:59:46] ENGINE Serving on http://172.21.15.152:8765 2024-04-26T04:59:47.580 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[39324]: [26/Apr/2024:04:59:46] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:59:47.580 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[39324]: [26/Apr/2024:04:59:46] ENGINE Bus STARTED 2024-04-26T04:59:47.580 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[39324]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:47.580 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:47 smithi152 ceph-mon[39324]: mgrmap e29: y(active, since 1.01039s) 2024-04-26T04:59:47.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:47 smithi195 ceph-mon[31427]: [26/Apr/2024:04:59:46] ENGINE Bus STARTING 2024-04-26T04:59:47.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:47 smithi195 ceph-mon[31427]: [26/Apr/2024:04:59:46] ENGINE Serving on http://172.21.15.152:8765 2024-04-26T04:59:47.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:47 smithi195 ceph-mon[31427]: [26/Apr/2024:04:59:46] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T04:59:47.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:47 smithi195 ceph-mon[31427]: [26/Apr/2024:04:59:46] ENGINE Bus STARTED 2024-04-26T04:59:47.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:47 smithi195 ceph-mon[31427]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:47.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:47 smithi195 ceph-mon[31427]: mgrmap e29: y(active, since 1.01039s) 2024-04-26T04:59:48.375 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:48.270+0000 7f081be9a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:59:48.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[27560]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3452496294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[27560]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:59:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[27560]: Cluster is now healthy 2024-04-26T04:59:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[39324]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3452496294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[39324]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:59:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:48 smithi152 ceph-mon[39324]: Cluster is now healthy 2024-04-26T04:59:48.634 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:48.443+0000 7f081be9a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:48.634 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-mon[31427]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:48.634 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3452496294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:48.635 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-mon[31427]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:59:48.635 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-mon[31427]: Cluster is now healthy 2024-04-26T04:59:49.200 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:48.633+0000 7f081be9a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:59:49.200 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:48.944+0000 7f081be9a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:59:49.200 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:49.040+0000 7f081be9a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:59:49.450 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:49.247+0000 7f081be9a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:59:49.450 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:49 smithi195 ceph-mon[31427]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:49.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:49 smithi152 ceph-mon[27560]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:49.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:49 smithi152 ceph-mon[39324]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:49.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:49 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:49.449+0000 7f081be9a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:50.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:50.102+0000 7f081be9a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:59:50.668 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:50.562+0000 7f081be9a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:59:50.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:50 smithi195 ceph-mon[31427]: mgrmap e30: y(active, since 3s) 2024-04-26T04:59:50.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:50 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3717630510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:50.669 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:50 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:50.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:50 smithi152 ceph-mon[27560]: mgrmap e30: y(active, since 3s) 2024-04-26T04:59:50.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:50 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3717630510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:50.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:50 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:50.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:50 smithi152 ceph-mon[39324]: mgrmap e30: y(active, since 3s) 2024-04-26T04:59:50.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:50 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3717630510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:50.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:50 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:50.667+0000 7f081be9a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:59:51.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:50.884+0000 7f081be9a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:59:51.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:51.444+0000 7f081be9a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:59:51.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:51.543+0000 7f081be9a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:59:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-mon[31427]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[27560]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[39324]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:51 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.167 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:51.800+0000 7f081be9a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:59:52.167 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:51.949+0000 7f081be9a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:59:52.167 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:52 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:52.049+0000 7f081be9a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:59:52.424 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:52 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:52.165+0000 7f081be9a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:52.424 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:52 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:52.260+0000 7f081be9a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:59:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:52 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.771 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:52 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-26T04:59:52.772 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:52 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:52.772 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:52 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:52.772 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:52 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.772 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:52 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.772 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:52 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:52.772 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:52 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:53.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:52 smithi195 podman[78064]: 2024-04-26 04:59:52.771260113 +0000 UTC m=+0.113738170 container create 52f6b6d0af48c20abe07581a836a67403a2fac2ae0b2bbf0eb10c4b79017870d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, version=8.5, io.openshift.expose-services=, io.buildah.version=1.32.0, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=Ceph Grafana Container, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:59:53.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:52 smithi195 podman[78064]: 2024-04-26 04:59:52.677272681 +0000 UTC m=+0.019750747 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:53.354 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:53.113+0000 7f081be9a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:59:53.354 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:53.353+0000 7f081be9a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:59:53.355 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 podman[78064]: 2024-04-26 04:59:53.065784081 +0000 UTC m=+0.408262138 container init 52f6b6d0af48c20abe07581a836a67403a2fac2ae0b2bbf0eb10c4b79017870d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, vcs-type=git, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, architecture=x86_64, name=ubi8) 2024-04-26T04:59:53.355 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 podman[78064]: 2024-04-26 04:59:53.071271715 +0000 UTC m=+0.413749772 container start 52f6b6d0af48c20abe07581a836a67403a2fac2ae0b2bbf0eb10c4b79017870d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, architecture=x86_64, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vendor=Red Hat, Inc., io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public) 2024-04-26T04:59:53.355 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 bash[78064]: 52f6b6d0af48c20abe07581a836a67403a2fac2ae0b2bbf0eb10c4b79017870d 2024-04-26T04:59:53.355 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:53.355 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[78081]: logger=settings t=2024-04-26T04:59:53.225982126Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:53.355 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 podman[78119]: 2024-04-26 04:59:53.255833159 +0000 UTC m=+0.022657912 container died 52f6b6d0af48c20abe07581a836a67403a2fac2ae0b2bbf0eb10c4b79017870d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, name=ubi8) 2024-04-26T04:59:53.494 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[27560]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3631128301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[27560]: Standby manager daemon x started 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[39324]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3631128301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[39324]: Standby manager daemon x started 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:53.495 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:53.496 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:53 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-mon[31427]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3631128301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-mon[31427]: Standby manager daemon x started 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1029191616' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:53] ENGINE Bus STARTING 2024-04-26T04:59:53.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: CherryPy Checker: 2024-04-26T04:59:53.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: The Application mounted at '' has an empty config. 2024-04-26T04:59:53.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: 2024-04-26T04:59:53.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:53] ENGINE Serving on http://:::9283 2024-04-26T04:59:53.782 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: [26/Apr/2024:04:59:53] ENGINE Bus STARTED 2024-04-26T04:59:54.280 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 podman[78119]: 2024-04-26 04:59:53.895140367 +0000 UTC m=+0.661965090 container remove 52f6b6d0af48c20abe07581a836a67403a2fac2ae0b2bbf0eb10c4b79017870d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, release=239.1651231664, version=8.5, vendor=Red Hat, Inc., description=Ceph Grafana Container, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8) 2024-04-26T04:59:54.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:53 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:54.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 04:59:54 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:55.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:54 smithi195 ceph-mon[31427]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:59:55.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:54 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:59:55.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:54 smithi152 ceph-mon[39324]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:59:55.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:54 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:59:55.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:54 smithi152 ceph-mon[27560]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:59:55.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:54 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:59:56.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:55 smithi195 ceph-mon[31427]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:59:56.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2470783178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:55 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:55 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:55 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:55 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:56.129 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[39324]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2470783178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[27560]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2470783178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:55 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[66612]: ::ffff:172.21.15.195 - - [26/Apr/2024:04:59:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:57.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:57.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:56 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:56 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T04:59:57.989 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:57 smithi195 ceph-mon[31427]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:57.989 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:57 smithi195 ceph-mon[31427]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:57.989 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:57 smithi195 ceph-mon[31427]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:59:57.989 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:57 smithi195 ceph-mon[31427]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:59:57.989 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:57 smithi195 ceph-mon[31427]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:57.989 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:57 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2517963508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:58.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[27560]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:58.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[27560]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:58.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[27560]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:59:58.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[27560]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:59:58.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[27560]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:58.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2517963508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:58.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[39324]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[39324]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[39324]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:59:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[39324]: Reconfiguring daemon prometheus.a on smithi195 2024-04-26T04:59:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[39324]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:57 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2517963508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:00.013 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 04:59:59 smithi195 ceph-mon[31427]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:00.013 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 04:59:59 smithi195 systemd[1]: Stopping Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:00.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 04:59:59 smithi152 ceph-mon[27560]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 04:59:59 smithi152 ceph-mon[39324]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.011Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T05:00:00.281 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T05:00:00.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T05:00:00.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T05:00:00.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T05:00:00.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[75518]: ts=2024-04-26T05:00:00.012Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T05:00:00.282 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 podman[79299]: 2024-04-26 05:00:00.02225969 +0000 UTC m=+0.122471036 container died 0bf2676a7051416d2b564317b156fc7f5d21fc3d4945cb5a57751560a92275fc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:00:00.707 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 podman[79299]: 2024-04-26 05:00:00.375684962 +0000 UTC m=+0.475896315 container cleanup 0bf2676a7051416d2b564317b156fc7f5d21fc3d4945cb5a57751560a92275fc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:00:00.707 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 bash[79299]: ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T05:00:01.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2021527464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:01.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-mon[31427]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-mon[31427]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:01.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:00 smithi195 ceph-mon[31427]: daemon grafana.a on smithi195 is in unknown state 2024-04-26T05:00:01.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 podman[79318]: 2024-04-26 05:00:00.786022257 +0000 UTC m=+0.765989231 container remove 0bf2676a7051416d2b564317b156fc7f5d21fc3d4945cb5a57751560a92275fc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:00:01.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T05:00:01.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 systemd[1]: Stopped Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:01.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.179s CPU time. 2024-04-26T05:00:01.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:00 smithi195 systemd[1]: Starting Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2021527464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[27560]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[27560]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[27560]: daemon grafana.a on smithi195 is in unknown state 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2021527464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[39324]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[39324]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:00 smithi152 ceph-mon[39324]: daemon grafana.a on smithi195 is in unknown state 2024-04-26T05:00:01.413 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 podman[79417]: 2024-04-26 05:00:01.056397002 +0000 UTC m=+0.019282819 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T05:00:01.413 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 podman[79417]: 2024-04-26 05:00:01.153573271 +0000 UTC m=+0.116459079 container create 153c7fab44cf92f6a35e7c32d3b845a879f0eab11d7f6c65975590f2e79fa36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:00:01.413 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 podman[79417]: 2024-04-26 05:00:01.364332699 +0000 UTC m=+0.327218506 container init 153c7fab44cf92f6a35e7c32d3b845a879f0eab11d7f6c65975590f2e79fa36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:00:01.413 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 podman[79417]: 2024-04-26 05:00:01.369853119 +0000 UTC m=+0.332738926 container start 153c7fab44cf92f6a35e7c32d3b845a879f0eab11d7f6c65975590f2e79fa36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:00:01.413 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.393Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.393Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.393Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi195 (none))" 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.393Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.393Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.394Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.395Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.396Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.396Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.398Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.398Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.53µs 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.398Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T05:00:01.414 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.398Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-26T05:00:01.415 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.412Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.414Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.414Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.414Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.934µs wal_replay_duration=15.783241ms wbl_replay_duration=114ns total_replay_duration=15.84121ms 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.416Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.416Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.417Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.434Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.998885ms db_storage=901ns remote_storage=1.188µs web_handler=348ns query_engine=1.042µs scrape=700.655µs scrape_sd=172.235µs notify=19.674µs notify_sd=21.258µs rules=15.856501ms tracing=5.59µs 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.434Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T05:00:01.709 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:01.434Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T05:00:01.710 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 bash[79417]: 153c7fab44cf92f6a35e7c32d3b845a879f0eab11d7f6c65975590f2e79fa36e 2024-04-26T05:00:01.710 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:01 smithi195 systemd[1]: Started Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:01.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:01] ENGINE Bus STOPPING 2024-04-26T05:00:01.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T05:00:01.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:01] ENGINE Bus STOPPED 2024-04-26T05:00:01.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:01] ENGINE Bus STARTING 2024-04-26T05:00:01.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:01] ENGINE Serving on http://:::9283 2024-04-26T05:00:01.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:01] ENGINE Bus STARTED 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:01.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:01.882 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:01 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:02.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:01 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:03.013 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:02 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T05:00:03.013 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:02 smithi195 ceph-mon[31427]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T05:00:03.013 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:02 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3492219225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:03.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:02 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T05:00:03.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:02 smithi152 ceph-mon[27560]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T05:00:03.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:02 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/3492219225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:03.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:02 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T05:00:03.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:02 smithi152 ceph-mon[39324]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T05:00:03.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:02 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3492219225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:03.896 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:03 smithi195 ceph-mon[31427]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:03.897 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:03 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:00:03.897 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:03 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:00:03.897 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:03 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:00:03.897 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:03 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:04.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[27560]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:04.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:00:04.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:00:04.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:00:04.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:04.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[39324]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T05:00:04.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:00:04.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:00:04.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:00:04.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:03 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:04.260 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:04 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-26T05:00:04.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:04 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:04.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:04 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:04.936 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:04 smithi195 ceph-mon[31427]: Upgrade: Updating mgr.x 2024-04-26T05:00:04.936 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:04 smithi195 ceph-mon[31427]: Deploying daemon mgr.x on smithi195 2024-04-26T05:00:04.936 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:04 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1166866742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:04 smithi152 ceph-mon[27560]: Upgrade: Updating mgr.x 2024-04-26T05:00:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:04 smithi152 ceph-mon[27560]: Deploying daemon mgr.x on smithi195 2024-04-26T05:00:05.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:04 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1166866742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:05.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:04 smithi152 ceph-mon[39324]: Upgrade: Updating mgr.x 2024-04-26T05:00:05.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:04 smithi152 ceph-mon[39324]: Deploying daemon mgr.x on smithi195 2024-04-26T05:00:05.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:04 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1166866742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:05.240 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:04 smithi195 systemd[1]: Stopping Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:05.241 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:04 smithi195 podman[79849]: 2024-04-26 05:00:04.849786104 +0000 UTC m=+0.021749495 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T05:00:05.241 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:04 smithi195 podman[79849]: 2024-04-26 05:00:04.954323657 +0000 UTC m=+0.126287030 container create b9372b8e2dc6af7cef394e3ff72711f831f81f4802b4926bad3cc1eebf3799e6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , release=239.1651231664, architecture=x86_64, description=Ceph Grafana Container, com.redhat.component=ubi8-container, version=8.5, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vcs-type=git, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, build-date=2022-04-29T11:48:11.055897) 2024-04-26T05:00:05.241 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:05 smithi195 podman[79849]: 2024-04-26 05:00:05.1570825 +0000 UTC m=+0.329045873 container init b9372b8e2dc6af7cef394e3ff72711f831f81f4802b4926bad3cc1eebf3799e6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.buildah.version=1.32.0, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , architecture=x86_64, name=ubi8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T05:00:05.241 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:05 smithi195 podman[79849]: 2024-04-26 05:00:05.162632806 +0000 UTC m=+0.334596172 container start b9372b8e2dc6af7cef394e3ff72711f831f81f4802b4926bad3cc1eebf3799e6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vcs-type=git, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , release=239.1651231664, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, vendor=Red Hat, Inc., io.buildah.version=1.32.0, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T05:00:05.241 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:05 smithi195 bash[79849]: b9372b8e2dc6af7cef394e3ff72711f831f81f4802b4926bad3cc1eebf3799e6 2024-04-26T05:00:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:05 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[79877]: logger=settings t=2024-04-26T05:00:05.323546061Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T05:00:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:05 smithi195 podman[79942]: 2024-04-26 05:00:05.406476933 +0000 UTC m=+0.072603974 container died b9372b8e2dc6af7cef394e3ff72711f831f81f4802b4926bad3cc1eebf3799e6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, vendor=Red Hat, Inc., distribution-scope=public, version=8.5, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, vcs-type=git, name=ubi8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T05:00:06.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:05 smithi195 ceph-mon[31427]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:06.031 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:05 smithi152 ceph-mon[27560]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:06.031 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:05 smithi152 ceph-mon[39324]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:06.431 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 podman[79929]: 2024-04-26 05:00:06.148584425 +0000 UTC m=+0.868685180 container died e4b994576e20711df72c338c6ac6023e296b5adeecd5963adfd99e5326440160 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T05:00:06.432 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:06 smithi195 podman[79942]: 2024-04-26 05:00:06.085784353 +0000 UTC m=+0.751911394 container remove b9372b8e2dc6af7cef394e3ff72711f831f81f4802b4926bad3cc1eebf3799e6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, name=ubi8, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, description=Ceph Grafana Container, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T05:00:06.432 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:06 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T05:00:06.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 podman[79929]: 2024-04-26 05:00:06.512969854 +0000 UTC m=+1.233070597 container cleanup e4b994576e20711df72c338c6ac6023e296b5adeecd5963adfd99e5326440160 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T05:00:06.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 bash[79929]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x 2024-04-26T05:00:07.133 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:06 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:00:07.133 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 podman[79961]: 2024-04-26 05:00:06.798239773 +0000 UTC m=+1.381644632 container remove e4b994576e20711df72c338c6ac6023e296b5adeecd5963adfd99e5326440160 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:00:07.133 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:00:07.133 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T05:00:07.133 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 systemd[1]: Stopped Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:07.134 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:06 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service: Consumed 26.020s CPU time. 2024-04-26T05:00:07.452 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 systemd[1]: Starting Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:07.452 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 podman[80136]: 2024-04-26 05:00:07.340872783 +0000 UTC m=+0.105137851 container create 178e4f8e339ac29bfe68380b7d5ff251a4aeafc614e6718f4d8fc04ef9904d8b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T05:00:07.452 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 podman[80136]: 2024-04-26 05:00:07.253948793 +0000 UTC m=+0.018213868 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:07.909 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:07 smithi195 ceph-mon[31427]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:07.909 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:07 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1834187577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:07.909 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 podman[80136]: 2024-04-26 05:00:07.551430993 +0000 UTC m=+0.315696061 container init 178e4f8e339ac29bfe68380b7d5ff251a4aeafc614e6718f4d8fc04ef9904d8b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:00:07.909 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 podman[80136]: 2024-04-26 05:00:07.557235788 +0000 UTC m=+0.321500853 container start 178e4f8e339ac29bfe68380b7d5ff251a4aeafc614e6718f4d8fc04ef9904d8b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T05:00:07.909 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 ceph-mgr[80173]: -- 172.21.15.195:0/3279663939 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5590b9db65a0 con 0x5590b9d8b000 2024-04-26T05:00:07.910 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 bash[80136]: 178e4f8e339ac29bfe68380b7d5ff251a4aeafc614e6718f4d8fc04ef9904d8b 2024-04-26T05:00:07.910 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 systemd[1]: Started Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:07.910 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:07.715+0000 7f433b4ce200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T05:00:07.910 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:07.907+0000 7f433b4ce200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T05:00:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:07 smithi152 ceph-mon[27560]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:07 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1834187577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:08.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:07 smithi152 ceph-mon[39324]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:08.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:07 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1834187577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:08.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:08.014+0000 7f433b4ce200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T05:00:08.604 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:08.347+0000 7f433b4ce200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T05:00:08.604 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:08.443+0000 7f433b4ce200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T05:00:09.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:09.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:08 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:09.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:09.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:08 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:10.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:09 smithi195 ceph-mon[31427]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:09 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2167016508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:10.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:09.698+0000 7f433b4ce200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T05:00:10.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:09.894+0000 7f433b4ce200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T05:00:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:09 smithi152 ceph-mon[27560]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:09 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/2167016508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:09 smithi152 ceph-mon[39324]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:09 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2167016508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:10.310 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:10.079+0000 7f433b4ce200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T05:00:10.711 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:10.391+0000 7f433b4ce200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T05:00:10.712 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:10.494+0000 7f433b4ce200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T05:00:11.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:10.710+0000 7f433b4ce200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T05:00:11.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:10.924+0000 7f433b4ce200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T05:00:12.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:11 smithi195 ceph-mon[31427]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:11 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:11.569+0000 7f433b4ce200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T05:00:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:11 smithi152 ceph-mon[27560]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:11 smithi152 ceph-mon[39324]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.352 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:12.031+0000 7f433b4ce200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T05:00:12.352 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:12.134+0000 7f433b4ce200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T05:00:12.352 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:12.350+0000 7f433b4ce200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T05:00:13.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:12 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:12 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/1478070225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:13.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:12 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:13.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:12 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:13.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:12 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1478070225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:13.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:12 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:13.263 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:12.904+0000 7f433b4ce200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T05:00:13.263 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:13.006+0000 7f433b4ce200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T05:00:13.264 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:12 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:13.264 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:12 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1478070225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:13.264 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:12 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:13.520 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:13.262+0000 7f433b4ce200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T05:00:13.520 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:13.417+0000 7f433b4ce200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T05:00:13.520 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:13.518+0000 7f433b4ce200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T05:00:13.780 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:13.642+0000 7f433b4ce200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T05:00:13.781 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:13.743+0000 7f433b4ce200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T05:00:14.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:13 smithi152 ceph-mon[27560]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:13 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:14.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:13 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:14.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:13 smithi152 ceph-mon[39324]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:13 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:14.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:13 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:14.266 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-mon[31427]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.266 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:14.266 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:13 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:14.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:00:14] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.43.0" 2024-04-26T05:00:14.778 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:14.590+0000 7f433b4ce200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/4209669003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-mon[31427]: Standby manager daemon x restarted 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-mon[31427]: Standby manager daemon x started 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T05:00:15.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:14.819+0000 7f433b4ce200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T05:00:15.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: [26/Apr/2024:05:00:14] ENGINE Bus STARTING 2024-04-26T05:00:15.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: CherryPy Checker: 2024-04-26T05:00:15.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: The Application mounted at '' has an empty config. 2024-04-26T05:00:15.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:15.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: [26/Apr/2024:05:00:14] ENGINE Serving on http://:::9283 2024-04-26T05:00:15.032 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: [26/Apr/2024:05:00:14] ENGINE Bus STARTED 2024-04-26T05:00:15.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/4209669003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:15.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[27560]: Standby manager daemon x restarted 2024-04-26T05:00:15.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[27560]: Standby manager daemon x started 2024-04-26T05:00:15.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T05:00:15.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T05:00:15.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T05:00:15.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[27560]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T05:00:15.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/4209669003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:15.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[39324]: Standby manager daemon x restarted 2024-04-26T05:00:15.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[39324]: Standby manager daemon x started 2024-04-26T05:00:15.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T05:00:15.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T05:00:15.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T05:00:15.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:14 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/1591397679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T05:00:16.190 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:15 smithi195 ceph-mon[31427]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.190 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:15 smithi195 ceph-mon[31427]: mgrmap e32: y(active, since 28s), standbys: x 2024-04-26T05:00:16.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:15 smithi152 ceph-mon[27560]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:15 smithi152 ceph-mon[27560]: mgrmap e32: y(active, since 28s), standbys: x 2024-04-26T05:00:16.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:15 smithi152 ceph-mon[39324]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:15 smithi152 ceph-mon[39324]: mgrmap e32: y(active, since 28s), standbys: x 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.153 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/855766747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.154 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T05:00:17.155 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:16 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/855766747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:17.267 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.267 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.267 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.267 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:16 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/855766747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:17.269 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-26T05:00:17.269 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:17.269 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:17.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 podman[83585]: 2024-04-26 05:00:17.266335418 +0000 UTC m=+0.114091478 container create f476d9847f13d21b5f00ae6d704c6724b6ec8bf6f018868adbb7b8717981642c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.openshift.tags=base rhel8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, name=ubi8, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container) 2024-04-26T05:00:17.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 podman[83585]: 2024-04-26 05:00:17.169661945 +0000 UTC m=+0.017417992 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T05:00:17.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 podman[83585]: 2024-04-26 05:00:17.510392512 +0000 UTC m=+0.358148527 container init f476d9847f13d21b5f00ae6d704c6724b6ec8bf6f018868adbb7b8717981642c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.tags=base rhel8, version=8.5, release=239.1651231664, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git) 2024-04-26T05:00:17.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 podman[83585]: 2024-04-26 05:00:17.516397911 +0000 UTC m=+0.364153927 container start f476d9847f13d21b5f00ae6d704c6724b6ec8bf6f018868adbb7b8717981642c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vendor=Red Hat, Inc., distribution-scope=public, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, version=8.5, architecture=x86_64) 2024-04-26T05:00:17.893 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 bash[83585]: f476d9847f13d21b5f00ae6d704c6724b6ec8bf6f018868adbb7b8717981642c 2024-04-26T05:00:17.893 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:17.893 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[83599]: logger=settings t=2024-04-26T05:00:17.665149938Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T05:00:17.893 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:17 smithi195 podman[83636]: 2024-04-26 05:00:17.693302025 +0000 UTC m=+0.018862377 container died f476d9847f13d21b5f00ae6d704c6724b6ec8bf6f018868adbb7b8717981642c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, io.openshift.expose-services=, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner ) 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[27560]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[27560]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[27560]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[27560]: Upgrade: Setting container_image for all mgr 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[27560]: Upgrade: It appears safe to stop mon.a 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[39324]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[39324]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[39324]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[39324]: Upgrade: Setting container_image for all mgr 2024-04-26T05:00:18.115 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:17 smithi152 ceph-mon[39324]: Upgrade: It appears safe to stop mon.a 2024-04-26T05:00:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:17 smithi195 ceph-mon[31427]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:17 smithi195 ceph-mon[31427]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:17 smithi195 ceph-mon[31427]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:17 smithi195 ceph-mon[31427]: Upgrade: Setting container_image for all mgr 2024-04-26T05:00:18.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:17 smithi195 ceph-mon[31427]: Upgrade: It appears safe to stop mon.a 2024-04-26T05:00:18.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:18 smithi195 podman[83636]: 2024-04-26 05:00:18.364722313 +0000 UTC m=+0.690282691 container remove f476d9847f13d21b5f00ae6d704c6724b6ec8bf6f018868adbb7b8717981642c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, name=ubi8, version=8.5, distribution-scope=public, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=) 2024-04-26T05:00:18.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:18 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T05:00:18.660 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:18 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:00:19.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: Upgrade: Updating mon.a 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: Deploying daemon mon.a on smithi152 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[27560]: from='client.? 172.21.15.152:0/369871693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:19.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: Upgrade: Updating mon.a 2024-04-26T05:00:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:00:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:00:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: Deploying daemon mon.a on smithi152 2024-04-26T05:00:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:19.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:19 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/369871693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:19.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: Upgrade: Updating mon.a 2024-04-26T05:00:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='mgr.y' 2024-04-26T05:00:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:00:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:00:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: from='mgr.25733 172.21.15.152:0/1296349722' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: Deploying daemon mon.a on smithi152 2024-04-26T05:00:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:19 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/369871693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:20.446 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:20 smithi152 systemd[1]: Stopping Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:20.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:20 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a[27537]: 2024-04-26T05:00:20.444+0000 7f7915c12700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:00:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:20 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a[27537]: 2024-04-26T05:00:20.444+0000 7f7915c12700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-26T05:00:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:20 smithi152 podman[101780]: 2024-04-26 05:00:20.690647471 +0000 UTC m=+0.393229080 container died 343ee858bd2cbcd665f81e8e9bba0279a42c802addc842d8de6d3d15b3b99b93 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:00:21.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:21 smithi152 podman[101780]: 2024-04-26 05:00:21.183583473 +0000 UTC m=+0.886165086 container cleanup 343ee858bd2cbcd665f81e8e9bba0279a42c802addc842d8de6d3d15b3b99b93 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T05:00:21.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:21 smithi152 bash[101780]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a 2024-04-26T05:00:21.813 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:21 smithi152 podman[101798]: 2024-04-26 05:00:21.468397562 +0000 UTC m=+0.773915221 container remove 343ee858bd2cbcd665f81e8e9bba0279a42c802addc842d8de6d3d15b3b99b93 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True) 2024-04-26T05:00:21.813 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:21 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T05:00:21.813 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:21 smithi152 systemd[1]: Stopped Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:21.813 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:21 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service: Consumed 12.233s CPU time. 2024-04-26T05:00:22.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:21 smithi152 systemd[1]: Starting Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:22.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 podman[101961]: 2024-04-26 05:00:21.931233382 +0000 UTC m=+0.018020696 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:22.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 podman[101961]: 2024-04-26 05:00:22.053129857 +0000 UTC m=+0.139917151 container create 1b4ec8125acddd75e92928f987b7ec1313aadc1647cab0ec97cbeeae5fb9feeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T05:00:22.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 podman[101961]: 2024-04-26 05:00:22.230397112 +0000 UTC m=+0.317184412 container init 1b4ec8125acddd75e92928f987b7ec1313aadc1647cab0ec97cbeeae5fb9feeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb) 2024-04-26T05:00:22.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 podman[101961]: 2024-04-26 05:00:22.235433659 +0000 UTC m=+0.322220949 container start 1b4ec8125acddd75e92928f987b7ec1313aadc1647cab0ec97cbeeae5fb9feeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS) 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 7 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: pidfile_write: ignore empty --pid-file 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: load: jerasure load: lrc 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: RocksDB version: 7.9.2 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Git sha 0 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: DB SUMMARY 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: DB Session ID: 8A4YEQEHDKKUZRQ72L5G 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: CURRENT file: CURRENT 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: MANIFEST file: MANIFEST-000015 size: 971 Bytes 2024-04-26T05:00:22.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000030.sst 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000028.log size: 2793889 ; 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.error_if_exists: 0 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.create_if_missing: 0 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.paranoid_checks: 1 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.env: 0x55d09b01bc60 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.info_log: 0x55d09d60faa0 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.statistics: (nil) 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.use_fsync: 0 2024-04-26T05:00:22.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_log_file_size: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.allow_fallocate: 1 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.use_direct_reads: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.db_log_dir: 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.wal_dir: 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T05:00:22.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.write_buffer_manager: 0x55d09d5e2820 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T05:00:22.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.unordered_write: 0 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.row_cache: None 2024-04-26T05:00:22.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.wal_filter: None 2024-04-26T05:00:22.636 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.two_write_queues: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.wal_compression: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.atomic_flush: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.log_readahead_size: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T05:00:22.637 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_background_jobs: 2 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_background_compactions: -1 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_subcompactions: 1 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T05:00:22.638 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_open_files: -1 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_background_flushes: -1 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Compression algorithms supported: 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kZSTD supported: 0 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kXpressCompression supported: 0 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kZlibCompression supported: 1 2024-04-26T05:00:22.639 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kSnappyCompression supported: 1 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kLZ4Compression supported: 1 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: kBZip2Compression supported: 0 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.merge_operator: 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_filter: None 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d09d60fba0) 2024-04-26T05:00:22.640 INFO:journalctl@ceph.mon.a.smithi152.stdout: cache_index_and_filter_blocks: 1 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: pin_top_level_index_and_filter: 1 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: index_type: 0 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: data_block_index_type: 0 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: index_shortening: 1 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: checksum: 4 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: no_block_cache: 0 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache: 0x55d09d613090 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache_name: BinnedLRUCache 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache_options: 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: capacity : 536870912 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: num_shard_bits : 4 2024-04-26T05:00:22.641 INFO:journalctl@ceph.mon.a.smithi152.stdout: strict_capacity_limit : 0 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: high_pri_pool_ratio: 0.000 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_cache_compressed: (nil) 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: persistent_cache: (nil) 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_size: 4096 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_size_deviation: 10 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_restart_interval: 16 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: index_block_restart_interval: 1 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: metadata_block_size: 4096 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: partition_filters: 0 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: use_delta_encoding: 1 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: filter_policy: bloomfilter 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: whole_key_filtering: 1 2024-04-26T05:00:22.642 INFO:journalctl@ceph.mon.a.smithi152.stdout: verify_compression: 0 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: read_amp_bytes_per_bit: 0 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: format_version: 5 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: enable_index_compression: 1 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: block_align: 0 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: max_auto_readahead_size: 262144 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: prepopulate_block_cache: 0 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: initial_auto_readahead_size: 8192 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression: NoCompression 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T05:00:22.643 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.num_levels: 7 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T05:00:22.644 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T05:00:22.645 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T05:00:22.646 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.table_properties_collectors: 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.inplace_update_support: 0 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T05:00:22.647 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.bloom_locality: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.max_successive_merges: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.ttl: 2592000 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.enable_blob_files: false 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.min_blob_size: 0 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T05:00:22.648 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 10642, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 4903732b-dbb9-4672-b8a2-8e997a85500a 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107622265156, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-26T05:00:22.649 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107622273133, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2538305, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10643, "largest_seqno": 12006, "table_properties": {"data_size": 2531479, "index_size": 4207, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 15219, "raw_average_key_size": 23, "raw_value_size": 2517664, "raw_average_value_size": 3964, "num_data_blocks": 193, "num_entries": 635, "num_filter_entries": 635, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107622, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "4903732b-dbb9-4672-b8a2-8e997a85500a", "db_session_id": "8A4YEQEHDKKUZRQ72L5G", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107622273233, "job": 1, "event": "recovery_finished"} 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d09d704000 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: DB pointer 0x55d09d6ee000 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** DB Stats ** 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:00:22.650 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: L0 1/0 2.42 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 609.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: L6 1/0 6.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: Sum 2/0 8.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 609.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 609.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T05:00:22.651 INFO:journalctl@ceph.mon.a.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 609.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: Cumulative compaction: 0.00 GB write, 228.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:00:22.652 INFO:journalctl@ceph.mon.a.smithi152.stdout: Interval compaction: 0.00 GB write, 228.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: Block cache BinnedLRUCache@0x55d09d613090#7 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: starting mon.a rank 0 at public addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] at bind addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???) e3 preinit fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???).mds e9 new map 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???).mds e9 print_map 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: e9 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: legacy client fscid: 1 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.653 INFO:journalctl@ceph.mon.a.smithi152.stdout: Filesystem 'foo' (1) 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: fs_name foo 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: epoch 8 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: created 2024-04-26T04:56:08.401096+0000 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: modified 2024-04-26T04:56:22.483807+0000 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: tableserver 0 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: root 0 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: session_timeout 60 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: session_autoclose 300 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: max_file_size 1099511627776 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: max_xattr_size 65536 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: required_client_features {} 2024-04-26T05:00:22.654 INFO:journalctl@ceph.mon.a.smithi152.stdout: last_failure 0 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: last_failure_osd_epoch 0 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: max_mds 1 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: in 0 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: up {0=14640} 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: failed 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: damaged 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: stopped 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: data_pools [4] 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: metadata_pool 3 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: inline_data disabled 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: balancer 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: bal_rank_mask -1 2024-04-26T05:00:22.655 INFO:journalctl@ceph.mon.a.smithi152.stdout: standby_count_wanted 1 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout: qdb_cluster leader: 0 members: 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout: [mds.foo.smithi152.rnhpxy{0:14640} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout: Standby daemons: 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout: 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout: [mds.foo.smithi195.bqeurw{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 ceph-mon[102000]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-04-26T05:00:22.656 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 bash[101961]: 1b4ec8125acddd75e92928f987b7ec1313aadc1647cab0ec97cbeeae5fb9feeb 2024-04-26T05:00:22.657 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:22 smithi152 systemd[1]: Started Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:24.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:00:24] "GET /metrics HTTP/1.1" 200 33852 "" "Prometheus/2.43.0" 2024-04-26T05:00:25.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ignoring --setuser ceph since I am not root 2024-04-26T05:00:25.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ignoring --setgroup ceph since I am not root 2024-04-26T05:00:25.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mgr[92667]: -- 172.21.15.152:0/1017853775 <== mon.0 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559362b405a0 con 0x559362b15000 2024-04-26T05:00:25.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:25.483+0000 7f70448c1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T05:00:25.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2757893456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:25.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: mon.a calling monitor election 2024-04-26T05:00:25.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3516667954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:25.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:00:25.632 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: monmap epoch 3 2024-04-26T05:00:25.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:25.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: last_changed 2024-04-26T04:50:48.501641+0000 2024-04-26T05:00:25.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:00:25.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: min_mon_release 17 (quincy) 2024-04-26T05:00:25.633 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: election_strategy: 1 2024-04-26T05:00:25.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:00:25.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:00:25.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:00:25.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:00:25.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T05:00:25.634 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-26T05:00:25.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:25.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:25.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: daemon grafana.a on smithi195 is in unknown state 2024-04-26T05:00:25.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: from='mgr.25733 ' entity='' 2024-04-26T05:00:25.635 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[102000]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2757893456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: mon.a calling monitor election 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/3516667954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:00:25.636 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: monmap epoch 3 2024-04-26T05:00:25.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:25.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: last_changed 2024-04-26T04:50:48.501641+0000 2024-04-26T05:00:25.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:00:25.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: min_mon_release 17 (quincy) 2024-04-26T05:00:25.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: election_strategy: 1 2024-04-26T05:00:25.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:00:25.637 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:00:25.638 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:00:25.638 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:00:25.638 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T05:00:25.638 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-26T05:00:25.638 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:25.638 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:25.638 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: daemon grafana.a on smithi195 is in unknown state 2024-04-26T05:00:25.639 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: from='mgr.25733 ' entity='' 2024-04-26T05:00:25.639 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-mon[39324]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: ignoring --setuser ceph since I am not root 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: ignoring --setgroup ceph since I am not root 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mgr[80173]: -- 172.21.15.195:0/1255680453 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a191fa85a0 con 0x55a191f7c800 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:25.482+0000 7f409d6c5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:25.663+0000 7f409d6c5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2757893456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:25.665 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: mon.a calling monitor election 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/3516667954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: monmap epoch 3 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: last_changed 2024-04-26T04:50:48.501641+0000 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: min_mon_release 17 (quincy) 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: election_strategy: 1 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:00:25.666 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: daemon grafana.a on smithi195 is in unknown state 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: from='mgr.25733 ' entity='' 2024-04-26T05:00:25.667 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-mon[31427]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T05:00:26.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:25.764+0000 7f409d6c5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T05:00:26.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:25.672+0000 7f70448c1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T05:00:26.102 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:25 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:25.778+0000 7f70448c1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T05:00:26.277 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-26T05:00:26.355 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:26 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:26.072+0000 7f409d6c5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T05:00:26.355 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:26 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:26.156+0000 7f409d6c5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T05:00:26.355 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:26 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:26.100+0000 7f70448c1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T05:00:26.355 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:26 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:26.188+0000 7f70448c1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T05:00:26.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:26 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1483140429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:26.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:26 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1483140429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:26.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:26 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1483140429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:26.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:00:27.264 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:27.677 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:27 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:27.334+0000 7f409d6c5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T05:00:27.677 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:27 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:27.504+0000 7f409d6c5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T05:00:27.713 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:27.420+0000 7f70448c1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T05:00:27.713 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:27.601+0000 7f70448c1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T05:00:27.965 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:27 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:27.676+0000 7f409d6c5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T05:00:28.072 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:27.778+0000 7f70448c1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T05:00:28.263 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:27 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:27.964+0000 7f409d6c5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T05:00:28.263 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:28.057+0000 7f409d6c5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T05:00:28.264 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:28.262+0000 7f409d6c5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T05:00:28.376 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:28.071+0000 7f70448c1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T05:00:28.376 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:28.166+0000 7f70448c1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T05:00:28.376 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:28.375+0000 7f70448c1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T05:00:28.530 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:28.457+0000 7f409d6c5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T05:00:28.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:28.582+0000 7f70448c1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T05:00:29.028 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:29.028 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:28 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-26T05:00:29.028 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:28 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:29.028 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:28 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:29.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:29 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:29.088+0000 7f409d6c5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T05:00:29.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 podman[83799]: 2024-04-26 05:00:28.929256448 +0000 UTC m=+0.017935212 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T05:00:29.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 podman[83799]: 2024-04-26 05:00:29.048834057 +0000 UTC m=+0.137512810 container create 0ccecdc043b4df758ee52043a24d8aacd6024b2965e89fcbbabbee3b96208fef (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, description=Ceph Grafana Container, distribution-scope=public, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, name=ubi8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.32.0, architecture=x86_64, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T05:00:29.281 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 podman[83799]: 2024-04-26 05:00:29.276209088 +0000 UTC m=+0.364887840 container init 0ccecdc043b4df758ee52043a24d8aacd6024b2965e89fcbbabbee3b96208fef (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vcs-type=git, com.redhat.component=ubi8-container, version=8.5, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , io.openshift.tags=base rhel8) 2024-04-26T05:00:29.546 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 podman[83799]: 2024-04-26 05:00:29.282022883 +0000 UTC m=+0.370701634 container start 0ccecdc043b4df758ee52043a24d8aacd6024b2965e89fcbbabbee3b96208fef (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, release=239.1651231664, version=8.5, maintainer=Paul Cuzner , name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.buildah.version=1.32.0) 2024-04-26T05:00:29.547 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 bash[83799]: 0ccecdc043b4df758ee52043a24d8aacd6024b2965e89fcbbabbee3b96208fef 2024-04-26T05:00:29.547 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:29.547 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[83815]: logger=settings t=2024-04-26T05:00:29.437322485Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T05:00:29.547 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:29 smithi195 podman[83854]: 2024-04-26 05:00:29.463408783 +0000 UTC m=+0.019037742 container died 0ccecdc043b4df758ee52043a24d8aacd6024b2965e89fcbbabbee3b96208fef (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., version=8.5, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T05:00:29.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:29 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:29.203+0000 7f70448c1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T05:00:29.851 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:29 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:29.545+0000 7f409d6c5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T05:00:29.851 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:29 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:29.643+0000 7f409d6c5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T05:00:29.938 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:29 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:29.633+0000 7f70448c1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T05:00:29.938 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:29 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:29.731+0000 7f70448c1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T05:00:29.938 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:29 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:29.937+0000 7f70448c1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T05:00:30.116 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:29 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:29.850+0000 7f409d6c5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T05:00:30.116 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:30 smithi195 podman[83854]: 2024-04-26 05:00:30.097432735 +0000 UTC m=+0.653061676 container remove 0ccecdc043b4df758ee52043a24d8aacd6024b2965e89fcbbabbee3b96208fef (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, io.buildah.version=1.32.0, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.openshift.tags=base rhel8, version=8.5, name=ubi8, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, com.redhat.component=ubi8-container) 2024-04-26T05:00:30.116 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:30 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T05:00:30.732 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:30 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:00:30.733 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:30.397+0000 7f409d6c5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T05:00:30.733 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:30.489+0000 7f409d6c5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T05:00:30.805 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:30 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:30.464+0000 7f70448c1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T05:00:30.806 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:30 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:30.558+0000 7f70448c1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T05:00:31.030 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:30.731+0000 7f409d6c5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T05:00:31.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:30.865+0000 7f409d6c5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T05:00:31.031 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:30 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:30.961+0000 7f409d6c5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T05:00:31.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:30 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:30.804+0000 7f70448c1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T05:00:31.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:30 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:30.940+0000 7f70448c1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T05:00:31.130 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:31 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:31.039+0000 7f70448c1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T05:00:31.530 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:31.078+0000 7f409d6c5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T05:00:31.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:31.173+0000 7f409d6c5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T05:00:31.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:31 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:31.157+0000 7f70448c1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T05:00:31.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:31 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:31.251+0000 7f70448c1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T05:00:32.263 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:31 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:31.982+0000 7f409d6c5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T05:00:32.263 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:32.201+0000 7f409d6c5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T05:00:32.263 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: [26/Apr/2024:05:00:32] ENGINE Bus STARTING 2024-04-26T05:00:32.379 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.070+0000 7f70448c1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T05:00:32.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.298+0000 7f70448c1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T05:00:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[102000]: Standby manager daemon x restarted 2024-04-26T05:00:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[102000]: Standby manager daemon x started 2024-04-26T05:00:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[102000]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T05:00:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[102000]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T05:00:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[102000]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T05:00:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[102000]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T05:00:32.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[39324]: Standby manager daemon x restarted 2024-04-26T05:00:32.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[39324]: Standby manager daemon x started 2024-04-26T05:00:32.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T05:00:32.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T05:00:32.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T05:00:32.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-mon[39324]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T05:00:32.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-mon[31427]: Standby manager daemon x restarted 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-mon[31427]: Standby manager daemon x started 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-mon[31427]: from='mgr.? 172.21.15.195:0/3617641874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: CherryPy Checker: 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: The Application mounted at '' has an empty config. 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: [26/Apr/2024:05:00:32] ENGINE Serving on http://:::9283 2024-04-26T05:00:32.531 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:00:32 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x[80150]: [26/Apr/2024:05:00:32] ENGINE Bus STARTED 2024-04-26T05:00:32.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:32] ENGINE Bus STARTING 2024-04-26T05:00:32.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.438+0000 7f6fa5c09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.438+0000 7f6fa5c09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.438+0000 7f6fa5c09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.438+0000 7f6fa5c09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.880 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.438+0000 7f6fa5c09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.443+0000 7f6fa2c03700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.443+0000 7f6fa2c03700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.443+0000 7f6fa2c03700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.443+0000 7f6fa2c03700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.443+0000 7f6fa2c03700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: CherryPy Checker: 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: The Application mounted at '' has an empty config. 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:32] ENGINE Serving on http://:::9283 2024-04-26T05:00:32.881 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:00:32] ENGINE Bus STARTED 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.a smithi152 *:9093,9094 running (84s) 42s ago 5m 29.0M - 0.25.0 c8568f914cd2 252d0e292dd9 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.a smithi195 *:3000 unknown 21s ago 5m - - 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi152.rnhpxy smithi152 running (4m) 42s ago 4m 15.0M - 17.2.7-902-gc61472c8 0de407c1b3af 6f0b0b86b7ef 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi195.bqeurw smithi195 running (4m) 21s ago 4m 15.0M - 17.2.7-902-gc61472c8 0de407c1b3af db58137dcdb3 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.x smithi195 *:8443,9283,8765 running (26s) 21s ago 9m 68.3M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 178e4f8e339a 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.y smithi152 *:8443,9283,8765 running (56s) 42s ago 11m 475M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 ec2ad8342f2f 2024-04-26T05:00:33.337 INFO:teuthology.orchestra.run.smithi152.stdout:mon.a smithi152 running (11m) 42s ago 11m 51.2M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 343ee858bd2c 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:mon.b smithi195 running (9m) 21s ago 9m 47.1M 2048M 17.2.7-902-gc61472c8 0de407c1b3af bf477f6a2b3d 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:mon.c smithi152 running (9m) 42s ago 9m 42.8M 2048M 17.2.7-902-gc61472c8 0de407c1b3af f907cd5ba87f 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.a smithi152 *:9100 running (81s) 42s ago 4m 8140k - 1.5.0 0da6a335fe13 8d4a4fd1dbae 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.b smithi195 *:9100 running (62s) 21s ago 4m 8330k - 1.5.0 0da6a335fe13 bdec586e73b3 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (9m) 42s ago 9m 48.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a6ef80972333 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (8m) 42s ago 8m 47.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 827c6f69d0b5 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (8m) 42s ago 8m 47.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3b4441be6ba1 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi152 running (7m) 42s ago 7m 45.2M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 1b39c722ab3d 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi195 running (7m) 21s ago 7m 47.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 958de1ceb551 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi195 running (7m) 21s ago 7m 45.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5a97e623bdc9 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.6 smithi195 running (6m) 21s ago 6m 44.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af f77aacbd5fd5 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:osd.7 smithi195 running (6m) 21s ago 6m 51.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1529e54096a 2024-04-26T05:00:33.338 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.a smithi195 *:9095 running (32s) 21s ago 5m 37.9M - 2.43.0 a07b618ecd1d 153c7fab44cf 2024-04-26T05:00:33.529 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:00:33.323+0000 7f7001d67700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-26T05:00:33.529 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: mgrmap e34: y(active, since 46s), standbys: x 2024-04-26T05:00:33.529 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: Active manager daemon y restarted 2024-04-26T05:00:33.529 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: Activating manager daemon y 2024-04-26T05:00:33.529 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:00:33.529 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: mgrmap e35: y(active, starting, since 0.0102389s), standbys: x 2024-04-26T05:00:33.529 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T05:00:33.530 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: Manager daemon y is now available 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:33.531 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: mgrmap e34: y(active, since 46s), standbys: x 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: Active manager daemon y restarted 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: Activating manager daemon y 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: mgrmap e35: y(active, starting, since 0.0102389s), standbys: x 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T05:00:33.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: Manager daemon y is now available 2024-04-26T05:00:33.533 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:33.534 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:33.534 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T05:00:33.534 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T05:00:33.534 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T05:00:33.534 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T05:00:33.534 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:33.534 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:33 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: mgrmap e34: y(active, since 46s), standbys: x 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: Active manager daemon y restarted 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: Activating manager daemon y 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: mgrmap e35: y(active, starting, since 0.0102389s), standbys: x 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.rnhpxy"}]: dispatch 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.bqeurw"}]: dispatch 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T05:00:33.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: Manager daemon y is now available 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:33.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T05:00:33.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T05:00:33.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T05:00:33.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T05:00:33.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:33.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:33 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:34.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T05:00:34.517 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:00:34] "GET /metrics HTTP/1.1" 200 31158 "" "Prometheus/2.43.0" 2024-04-26T05:00:34.517 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[102000]: [26/Apr/2024:05:00:32] ENGINE Bus STARTING 2024-04-26T05:00:34.517 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[102000]: [26/Apr/2024:05:00:32] ENGINE Serving on http://172.21.15.152:8765 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[102000]: [26/Apr/2024:05:00:32] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[102000]: [26/Apr/2024:05:00:32] ENGINE Bus STARTED 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[102000]: mgrmap e36: y(active, since 1.01944s), standbys: x 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[102000]: from='client.25913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[102000]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[39324]: [26/Apr/2024:05:00:32] ENGINE Bus STARTING 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[39324]: [26/Apr/2024:05:00:32] ENGINE Serving on http://172.21.15.152:8765 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[39324]: [26/Apr/2024:05:00:32] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[39324]: [26/Apr/2024:05:00:32] ENGINE Bus STARTED 2024-04-26T05:00:34.518 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[39324]: mgrmap e36: y(active, since 1.01944s), standbys: x 2024-04-26T05:00:34.519 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[39324]: from='client.25913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:34.519 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:34 smithi152 ceph-mon[39324]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:34 smithi195 ceph-mon[31427]: [26/Apr/2024:05:00:32] ENGINE Bus STARTING 2024-04-26T05:00:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:34 smithi195 ceph-mon[31427]: [26/Apr/2024:05:00:32] ENGINE Serving on http://172.21.15.152:8765 2024-04-26T05:00:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:34 smithi195 ceph-mon[31427]: [26/Apr/2024:05:00:32] ENGINE Serving on https://172.21.15.152:7150 2024-04-26T05:00:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:34 smithi195 ceph-mon[31427]: [26/Apr/2024:05:00:32] ENGINE Bus STARTED 2024-04-26T05:00:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:34 smithi195 ceph-mon[31427]: mgrmap e36: y(active, since 1.01944s), standbys: x 2024-04-26T05:00:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:34 smithi195 ceph-mon[31427]: from='client.25913 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:34 smithi195 ceph-mon[31427]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.941 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:35.537 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[102000]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T05:00:35.537 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[102000]: Cluster is now healthy 2024-04-26T05:00:35.537 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[102000]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:35.537 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[102000]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T05:00:35.537 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[39324]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T05:00:35.537 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[39324]: Cluster is now healthy 2024-04-26T05:00:35.537 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[39324]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:35.538 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:35 smithi152 ceph-mon[39324]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T05:00:35.757 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:35 smithi195 ceph-mon[31427]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T05:00:35.757 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:35 smithi195 ceph-mon[31427]: Cluster is now healthy 2024-04-26T05:00:35.757 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:35 smithi195 ceph-mon[31427]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:35.757 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:35 smithi195 ceph-mon[31427]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T05:00:37.343 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[39324]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:37.343 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[39324]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T05:00:37.343 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:37.344 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:37.344 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[102000]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:37.344 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[102000]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T05:00:37.344 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:37.344 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:37 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:37.613 INFO:teuthology.orchestra.run.smithi152.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:37.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:37 smithi195 ceph-mon[31427]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:37.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:37 smithi195 ceph-mon[31427]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T05:00:37.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:37 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:37.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:37 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.818 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[102000]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[102000]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[102000]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[39324]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[39324]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.819 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:38 smithi152 ceph-mon[39324]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:38.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T05:00:38.863 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:38 smithi195 ceph-mon[31427]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:38.863 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:38 smithi195 ceph-mon[31427]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:38.864 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:38 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.864 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:38 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.864 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:38 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:38.864 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:38 smithi195 ceph-mon[31427]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:38.864 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:39.544 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:40.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:40 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-26T05:00:40.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:40 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:40.808 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:40 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:40.808 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:40 smithi195 podman[85067]: 2024-04-26 05:00:40.676302784 +0000 UTC m=+0.017038315 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T05:00:41.068 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:40 smithi195 podman[85067]: 2024-04-26 05:00:40.807199499 +0000 UTC m=+0.147935016 container create ef3c1eaafd5175fcf91c28bc1a7fb570772421ad80aabe2e8b68decfd89d19ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, name=ubi8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, distribution-scope=public, vcs-type=git) 2024-04-26T05:00:41.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:40 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:41.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:40 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:41.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:40 smithi195 ceph-mon[31427]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T05:00:41.086 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:40 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:41.086 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:40 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:41.086 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:40 smithi152 ceph-mon[39324]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T05:00:41.086 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:40 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:41.086 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:40 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:41.086 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:40 smithi152 ceph-mon[102000]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T05:00:41.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:41 smithi195 podman[85067]: 2024-04-26 05:00:41.068167506 +0000 UTC m=+0.408903026 container init ef3c1eaafd5175fcf91c28bc1a7fb570772421ad80aabe2e8b68decfd89d19ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.openshift.tags=base rhel8, architecture=x86_64, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, version=8.5, com.redhat.component=ubi8-container, name=ubi8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T05:00:41.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:41 smithi195 podman[85067]: 2024-04-26 05:00:41.074309819 +0000 UTC m=+0.415045340 container start ef3c1eaafd5175fcf91c28bc1a7fb570772421ad80aabe2e8b68decfd89d19ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, architecture=x86_64, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.expose-services=, name=ubi8) 2024-04-26T05:00:41.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:41 smithi195 bash[85067]: ef3c1eaafd5175fcf91c28bc1a7fb570772421ad80aabe2e8b68decfd89d19ec 2024-04-26T05:00:41.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:41 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:41.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:41 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[86609]: logger=settings t=2024-04-26T05:00:41.225000757Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T05:00:42.131 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:41 smithi195 podman[87655]: 2024-04-26 05:00:41.881988308 +0000 UTC m=+0.649770462 container died ef3c1eaafd5175fcf91c28bc1a7fb570772421ad80aabe2e8b68decfd89d19ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, release=239.1651231664, vendor=Red Hat, Inc., version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, description=Ceph Grafana Container, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-type=git, com.redhat.component=ubi8-container) 2024-04-26T05:00:42.456 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:42 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.456 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:42 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.456 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:42 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.456 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:42 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.456 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:42 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:42.456 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:42 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:42.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:42 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:42.780 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:42 smithi195 podman[87655]: 2024-04-26 05:00:42.552861879 +0000 UTC m=+1.320644040 container remove ef3c1eaafd5175fcf91c28bc1a7fb570772421ad80aabe2e8b68decfd89d19ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, name=ubi8, release=239.1651231664, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, architecture=x86_64, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T05:00:42.780 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:42 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T05:00:43.135 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:42 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:43.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:43 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.conf 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:43.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:43.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:43.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:43.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:43 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T05:00:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:44 smithi195 ceph-mon[31427]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T05:00:44.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:00:44] "GET /metrics HTTP/1.1" 200 31158 "" "Prometheus/2.43.0" 2024-04-26T05:00:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T05:00:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T05:00:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T05:00:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[102000]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: Updating smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: Updating smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T05:00:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:44 smithi152 ceph-mon[39324]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T05:00:46.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:46 smithi195 ceph-mon[31427]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T05:00:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:46 smithi152 ceph-mon[102000]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T05:00:46.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:46 smithi152 ceph-mon[39324]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T05:00:48.454 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:48 smithi152 ceph-mon[102000]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:48.454 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:48.455 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:48 smithi152 ceph-mon[39324]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:48.455 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:48 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:48 smithi195 ceph-mon[31427]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:48.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:48 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:49.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:50 smithi195 ceph-mon[31427]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:50 smithi152 ceph-mon[102000]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:50.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:50 smithi152 ceph-mon[39324]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:52.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:52 smithi195 ceph-mon[31427]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:52.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:52 smithi152 ceph-mon[102000]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:52.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:52 smithi152 ceph-mon[39324]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:53.359 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-26T05:00:53.359 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:53.360 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:53.360 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 podman[88672]: 2024-04-26 05:00:53.174407935 +0000 UTC m=+0.018141495 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T05:00:53.632 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 podman[88672]: 2024-04-26 05:00:53.396008373 +0000 UTC m=+0.239741926 container create 99cedc1c102aa013c58d40532f7d82de83f7e688151c0c948039a3d672eb747d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.openshift.expose-services=) 2024-04-26T05:00:53.632 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 podman[88672]: 2024-04-26 05:00:53.631682066 +0000 UTC m=+0.475415616 container init 99cedc1c102aa013c58d40532f7d82de83f7e688151c0c948039a3d672eb747d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, vcs-type=git, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, release=239.1651231664, maintainer=Paul Cuzner , distribution-scope=public, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T05:00:54.030 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 podman[88672]: 2024-04-26 05:00:53.637338972 +0000 UTC m=+0.481072523 container start 99cedc1c102aa013c58d40532f7d82de83f7e688151c0c948039a3d672eb747d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , distribution-scope=public, version=8.5, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, name=ubi8, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, description=Ceph Grafana Container, vcs-type=git, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc.) 2024-04-26T05:00:54.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 bash[88672]: 99cedc1c102aa013c58d40532f7d82de83f7e688151c0c948039a3d672eb747d 2024-04-26T05:00:54.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:54.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[88687]: logger=settings t=2024-04-26T05:00:53.785871962Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T05:00:54.031 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:53 smithi195 podman[88722]: 2024-04-26 05:00:53.812522588 +0000 UTC m=+0.019226359 container died 99cedc1c102aa013c58d40532f7d82de83f7e688151c0c948039a3d672eb747d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.expose-services=, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, architecture=x86_64, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897) 2024-04-26T05:00:54.352 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:54 smithi195 ceph-mon[31427]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:54 smithi152 ceph-mon[102000]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:54 smithi152 ceph-mon[39324]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:54.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:00:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:00:54] "GET /metrics HTTP/1.1" 200 33849 "" "Prometheus/2.43.0" 2024-04-26T05:00:54.780 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:54 smithi195 podman[88722]: 2024-04-26 05:00:54.352112896 +0000 UTC m=+0.558816719 container remove 99cedc1c102aa013c58d40532f7d82de83f7e688151c0c948039a3d672eb747d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., description=Ceph Grafana Container, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, name=ubi8, release=239.1651231664) 2024-04-26T05:00:54.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:54 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T05:00:54.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:00:54 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:00:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:56 smithi195 ceph-mon[31427]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:56.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:56 smithi152 ceph-mon[102000]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:56.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:56 smithi152 ceph-mon[39324]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:58.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:00:58 smithi195 ceph-mon[31427]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:58.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:00:58 smithi152 ceph-mon[102000]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:58.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:00:58 smithi152 ceph-mon[39324]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:59.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:00:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:00:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:00.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:00 smithi195 ceph-mon[31427]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:00.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:00 smithi152 ceph-mon[102000]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:00.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:00 smithi152 ceph-mon[39324]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:02.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:02 smithi195 ceph-mon[31427]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:02.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:02 smithi152 ceph-mon[102000]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:02.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:02 smithi152 ceph-mon[39324]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:03 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:03.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:03.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:03 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:04 smithi195 ceph-mon[31427]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:04 smithi152 ceph-mon[102000]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:04 smithi152 ceph-mon[39324]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:04.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:01:04] "GET /metrics HTTP/1.1" 200 33848 "" "Prometheus/2.43.0" 2024-04-26T05:01:05.141 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:04 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-26T05:01:05.142 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:04 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:05.142 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:04 smithi195 systemd[1]: Starting Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:01:05.142 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 podman[88840]: 2024-04-26 05:01:04.915210194 +0000 UTC m=+0.016965947 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T05:01:05.142 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 podman[88840]: 2024-04-26 05:01:05.027691691 +0000 UTC m=+0.129447433 container create c4c737d55f1efd24a9a9a6775e37a5ab4ee6caeddbb19f41864b988dac827fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, distribution-scope=public, io.openshift.tags=base rhel8, name=ubi8, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, release=239.1651231664) 2024-04-26T05:01:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 podman[88840]: 2024-04-26 05:01:05.263465243 +0000 UTC m=+0.365220988 container init c4c737d55f1efd24a9a9a6775e37a5ab4ee6caeddbb19f41864b988dac827fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, architecture=x86_64, io.openshift.tags=base rhel8, release=239.1651231664, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, name=ubi8, build-date=2022-04-29T11:48:11.055897, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T05:01:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 podman[88840]: 2024-04-26 05:01:05.269257029 +0000 UTC m=+0.371012770 container start c4c737d55f1efd24a9a9a6775e37a5ab4ee6caeddbb19f41864b988dac827fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, release=239.1651231664, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.expose-services=) 2024-04-26T05:01:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 bash[88840]: c4c737d55f1efd24a9a9a6775e37a5ab4ee6caeddbb19f41864b988dac827fcb 2024-04-26T05:01:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 systemd[1]: Started Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a[88855]: logger=settings t=2024-04-26T05:01:05.416675706Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T05:01:05.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:05 smithi195 podman[88891]: 2024-04-26 05:01:05.442649765 +0000 UTC m=+0.018735076 container died c4c737d55f1efd24a9a9a6775e37a5ab4ee6caeddbb19f41864b988dac827fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, release=239.1651231664, description=Ceph Grafana Container, distribution-scope=public, version=8.5, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, maintainer=Paul Cuzner ) 2024-04-26T05:01:06.432 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:06 smithi152 ceph-mon[102000]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:06.432 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:06 smithi152 ceph-mon[39324]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:06.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:06 smithi195 ceph-mon[31427]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:06.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:06 smithi195 podman[88891]: 2024-04-26 05:01:06.034105222 +0000 UTC m=+0.610190568 container remove c4c737d55f1efd24a9a9a6775e37a5ab4ee6caeddbb19f41864b988dac827fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., architecture=x86_64, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, release=239.1651231664, io.openshift.expose-services=, description=Ceph Grafana Container, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0) 2024-04-26T05:01:06.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:06 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T05:01:06.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:06 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:01:08.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:08 smithi195 ceph-mon[31427]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:08.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:08 smithi152 ceph-mon[102000]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:08.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:08 smithi152 ceph-mon[39324]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:01:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:01:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:10.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:10 smithi195 ceph-mon[31427]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:10.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:10 smithi152 ceph-mon[102000]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:10.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:10 smithi152 ceph-mon[39324]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:12.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:12 smithi195 ceph-mon[31427]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:12.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:12 smithi152 ceph-mon[102000]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:12.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:12 smithi152 ceph-mon[39324]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:14.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:14 smithi195 ceph-mon[31427]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:14.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:14 smithi152 ceph-mon[102000]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:14.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:14 smithi152 ceph-mon[39324]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:14.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:01:14] "GET /metrics HTTP/1.1" 200 33848 "" "Prometheus/2.43.0" 2024-04-26T05:01:16.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:16 smithi195 ceph-mon[31427]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:16.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:16 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-04-26T05:01:16.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:16 smithi195 systemd[1]: Stopped Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:16.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:16 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Start request repeated too quickly. 2024-04-26T05:01:16.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:16 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:01:16.531 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Apr 26 05:01:16 smithi195 systemd[1]: Failed to start Ceph grafana.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:16.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:16 smithi152 ceph-mon[102000]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:16.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:16 smithi152 ceph-mon[39324]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:18 smithi152 ceph-mon[102000]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:18.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:18 smithi152 ceph-mon[39324]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:18 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:18.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:18 smithi195 ceph-mon[31427]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:18 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:19.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:01:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:01:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:20.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:20 smithi152 ceph-mon[102000]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:20.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:20 smithi152 ceph-mon[39324]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:20.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:20 smithi195 ceph-mon[31427]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:22 smithi152 ceph-mon[102000]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:22 smithi152 ceph-mon[39324]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:22 smithi195 ceph-mon[31427]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:24.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:24 smithi152 ceph-mon[102000]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:24.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:24 smithi152 ceph-mon[39324]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:24.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:01:24] "GET /metrics HTTP/1.1" 200 33842 "" "Prometheus/2.43.0" 2024-04-26T05:01:24.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:24 smithi195 ceph-mon[31427]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:26 smithi152 ceph-mon[102000]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:26 smithi152 ceph-mon[39324]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:26 smithi195 ceph-mon[31427]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:28 smithi152 ceph-mon[102000]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:28 smithi152 ceph-mon[39324]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:28 smithi195 ceph-mon[31427]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:29.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:01:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:01:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:30.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:30 smithi152 ceph-mon[39324]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:30.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:30 smithi152 ceph-mon[102000]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:30.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:30 smithi195 ceph-mon[31427]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:32.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:32 smithi152 ceph-mon[102000]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:32 smithi152 ceph-mon[39324]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:32.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:32 smithi195 ceph-mon[31427]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:33.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:33.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:33 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:33.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:33 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:34.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:34 smithi152 ceph-mon[102000]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:34.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:34 smithi152 ceph-mon[39324]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:34.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:01:34] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2024-04-26T05:01:34.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:34 smithi195 ceph-mon[31427]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:36 smithi152 ceph-mon[102000]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:36 smithi152 ceph-mon[39324]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:36 smithi195 ceph-mon[31427]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:38 smithi152 ceph-mon[39324]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:38 smithi152 ceph-mon[102000]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:38 smithi195 ceph-mon[31427]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:39.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:01:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:01:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:40 smithi152 ceph-mon[39324]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:40.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:40 smithi152 ceph-mon[102000]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:40.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:40 smithi195 ceph-mon[31427]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:41.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T05:01:42.230 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:42 smithi152 ceph-mon[102000]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:42 smithi152 ceph-mon[39324]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:42.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:42 smithi195 ceph-mon[31427]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:43 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:43.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:43 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:43.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:43 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:44.339 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:01:44] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2024-04-26T05:01:44.339 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:44 smithi152 ceph-mon[102000]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:44.433 INFO:teuthology.orchestra.run.smithi152.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:44.629 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:44 smithi152 ceph-mon[39324]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:44.654 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:44 smithi195 ceph-mon[31427]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:45.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-26T05:01:45.765 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:45 smithi195 ceph-mon[31427]: from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:45.765 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:45 smithi195 ceph-mon[31427]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:45.765 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:45 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:45.765 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:45 smithi195 ceph-mon[31427]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[102000]: from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[102000]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[102000]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[39324]: from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[39324]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:45.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:45 smithi152 ceph-mon[39324]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:46.036 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:48.194 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[102000]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.194 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:48.194 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:48.194 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:48.195 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[39324]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.195 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:48.195 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:48.195 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:48 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:48.427 INFO:teuthology.orchestra.run.smithi152.stdout:wait for majority of mons to upgrade 2024-04-26T05:01:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:48 smithi195 ceph-mon[31427]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:48 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:48.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:48 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:48.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:48 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:49.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:01:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:01:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:49.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-26T05:01:49.429 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:50.229 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:50 smithi152 ceph-mon[102000]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:50.229 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:50 smithi152 ceph-mon[39324]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:50 smithi195 ceph-mon[31427]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[102000]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2387080733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[39324]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:52.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:52 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/2387080733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:52 smithi195 ceph-mon[31427]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:52 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:52 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:52 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/2387080733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:53.182 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T05:01:53.183 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[39324]: Upgrade: It appears safe to stop mon.c 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T05:01:53.184 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:53 smithi152 ceph-mon[102000]: Upgrade: It appears safe to stop mon.c 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T05:01:53.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:53 smithi195 ceph-mon[31427]: Upgrade: It appears safe to stop mon.c 2024-04-26T05:01:54.379 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:01:54] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2024-04-26T05:01:55.368 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: Upgrade: Updating mon.c 2024-04-26T05:01:55.368 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:55.368 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:55.368 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:55.368 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: Deploying daemon mon.c on smithi152 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[39324]: from='client.? 172.21.15.152:0/1760217522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: Upgrade: Updating mon.c 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: Deploying daemon mon.c on smithi152 2024-04-26T05:01:55.369 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:55.370 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:01:55 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1760217522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: Upgrade: Updating mon.c 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: from='mgr.25898 ' entity='mgr.y' 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: Deploying daemon mon.c on smithi152 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:01:55 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1760217522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:56 smithi152 systemd[1]: Stopping Ceph mon.c for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:01:56.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c[39301]: 2024-04-26T05:01:56.307+0000 7f3a7cd9c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:01:56.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c[39301]: 2024-04-26T05:01:56.307+0000 7f3a7cd9c700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-26T05:01:56.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:56 smithi152 podman[106309]: 2024-04-26 05:01:56.328976573 +0000 UTC m=+0.152039875 container died f907cd5ba87f7c9bb45ae6a9ed85044eb5bd52548e227e770c27314e8d5243da (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T05:01:56.625 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:01:56] ENGINE Bus STOPPING 2024-04-26T05:01:56.625 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:01:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T05:01:56.625 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:01:56] ENGINE Bus STOPPED 2024-04-26T05:01:56.625 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:01:56] ENGINE Bus STARTING 2024-04-26T05:01:56.625 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:01:56] ENGINE Serving on http://:::9283 2024-04-26T05:01:56.625 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:01:56 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:01:56] ENGINE Bus STARTED 2024-04-26T05:01:56.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:56 smithi152 podman[106309]: 2024-04-26 05:01:56.845357788 +0000 UTC m=+0.668421087 container cleanup f907cd5ba87f7c9bb45ae6a9ed85044eb5bd52548e227e770c27314e8d5243da (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T05:01:56.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:56 smithi152 bash[106309]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c 2024-04-26T05:01:57.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 podman[106328]: 2024-04-26 05:01:57.074051332 +0000 UTC m=+0.747169854 container remove f907cd5ba87f7c9bb45ae6a9ed85044eb5bd52548e227e770c27314e8d5243da (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T05:01:57.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-26T05:01:57.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 systemd[1]: Stopped Ceph mon.c for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:57.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.c.service: Consumed 7.847s CPU time. 2024-04-26T05:01:57.761 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 systemd[1]: Starting Ceph mon.c for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:01:57.761 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 podman[106501]: 2024-04-26 05:01:57.530195132 +0000 UTC m=+0.018228077 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:57.762 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 podman[106501]: 2024-04-26 05:01:57.64225725 +0000 UTC m=+0.130290164 container create 0bfb660a4a44fa6e6b8f02247942cb4c6942e91ca8289bf4ed5d03382c7df4d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb) 2024-04-26T05:01:58.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 podman[106501]: 2024-04-26 05:01:57.853098985 +0000 UTC m=+0.341131904 container init 0bfb660a4a44fa6e6b8f02247942cb4c6942e91ca8289bf4ed5d03382c7df4d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T05:01:58.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 podman[106501]: 2024-04-26 05:01:57.858567464 +0000 UTC m=+0.346600378 container start 0bfb660a4a44fa6e6b8f02247942cb4c6942e91ca8289bf4ed5d03382c7df4d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 7 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: pidfile_write: ignore empty --pid-file 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: load: jerasure load: lrc 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: RocksDB version: 7.9.2 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Git sha 0 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: DB SUMMARY 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: DB Session ID: PHT0H45EYR1D9MS0QQBD 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: CURRENT file: CURRENT 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T05:01:58.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 2851456 ; 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.error_if_exists: 0 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.create_if_missing: 0 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.paranoid_checks: 1 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.env: 0x55a6973b9c60 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.info_log: 0x55a69993daa0 2024-04-26T05:01:58.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T05:01:58.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.statistics: (nil) 2024-04-26T05:01:58.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.use_fsync: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_log_file_size: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.allow_fallocate: 1 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.use_direct_reads: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.db_log_dir: 2024-04-26T05:01:58.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.wal_dir: 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.write_buffer_manager: 0x55a699910820 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T05:01:58.135 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.unordered_write: 0 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.row_cache: None 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.wal_filter: None 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T05:01:58.136 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.two_write_queues: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.wal_compression: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.atomic_flush: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.log_readahead_size: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T05:01:58.137 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_background_jobs: 2 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_background_compactions: -1 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_subcompactions: 1 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T05:01:58.138 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_open_files: -1 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_background_flushes: -1 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Compression algorithms supported: 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kZSTD supported: 0 2024-04-26T05:01:58.139 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kXpressCompression supported: 0 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kZlibCompression supported: 1 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kSnappyCompression supported: 1 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kLZ4Compression supported: 1 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: kBZip2Compression supported: 0 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-26T05:01:58.140 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.merge_operator: 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_filter: None 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a69993dba0) 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout: cache_index_and_filter_blocks: 1 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T05:01:58.141 INFO:journalctl@ceph.mon.c.smithi152.stdout: pin_top_level_index_and_filter: 1 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: index_type: 0 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: data_block_index_type: 0 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: index_shortening: 1 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: checksum: 4 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: no_block_cache: 0 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache: 0x55a699941090 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache_name: BinnedLRUCache 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache_options: 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: capacity : 536870912 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: num_shard_bits : 4 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: strict_capacity_limit : 0 2024-04-26T05:01:58.142 INFO:journalctl@ceph.mon.c.smithi152.stdout: high_pri_pool_ratio: 0.000 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_cache_compressed: (nil) 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: persistent_cache: (nil) 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_size: 4096 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_size_deviation: 10 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_restart_interval: 16 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: index_block_restart_interval: 1 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: metadata_block_size: 4096 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: partition_filters: 0 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: use_delta_encoding: 1 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: filter_policy: bloomfilter 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: whole_key_filtering: 1 2024-04-26T05:01:58.143 INFO:journalctl@ceph.mon.c.smithi152.stdout: verify_compression: 0 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: read_amp_bytes_per_bit: 0 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: format_version: 5 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: enable_index_compression: 1 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: block_align: 0 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: max_auto_readahead_size: 262144 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: prepopulate_block_cache: 0 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: initial_auto_readahead_size: 8192 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T05:01:58.144 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression: NoCompression 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.num_levels: 7 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T05:01:58.145 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:01:58.146 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T05:01:58.147 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T05:01:58.148 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T05:01:58.148 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T05:01:58.148 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T05:01:58.148 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T05:01:58.148 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T05:01:58.149 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.table_properties_collectors: 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.inplace_update_support: 0 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.bloom_locality: 0 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.max_successive_merges: 0 2024-04-26T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.ttl: 2592000 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.enable_blob_files: false 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.min_blob_size: 0 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12987, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 91721876-f8ac-4a2b-bf45-added091b872 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107717888576, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107717897666, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1785821, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12992, "largest_seqno": 14071, "table_properties": {"data_size": 1780574, "index_size": 3144, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1093, "raw_key_size": 9298, "raw_average_key_size": 22, "raw_value_size": 1771277, "raw_average_value_size": 4237, "num_data_blocks": 144, "num_entries": 418, "num_filter_entries": 418, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107717, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "91721876-f8ac-4a2b-bf45-added091b872", "db_session_id": "PHT0H45EYR1D9MS0QQBD", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107717897792, "job": 1, "event": "recovery_finished"} 2024-04-26T05:01:58.152 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a699a32000 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: DB pointer 0x55a699a1c000 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** DB Stats ** 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T05:01:58.153 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: L0 1/0 1.70 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 546.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: L6 1/0 7.82 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: Sum 2/0 9.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 546.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 546.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:01:58.154 INFO:journalctl@ceph.mon.c.smithi152.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 546.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: Cumulative compaction: 0.00 GB write, 133.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: Interval compaction: 0.00 GB write, 133.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T05:01:58.155 INFO:journalctl@ceph.mon.c.smithi152.stdout: Block cache BinnedLRUCache@0x55a699941090#7 capacity: 512.00 MB usage: 4.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: starting mon.c rank 2 at public addrs [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] at bind addrs [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???) e3 preinit fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???).mds e9 new map 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???).mds e9 print_map 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: e9 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: legacy client fscid: 1 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: Filesystem 'foo' (1) 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: fs_name foo 2024-04-26T05:01:58.156 INFO:journalctl@ceph.mon.c.smithi152.stdout: epoch 8 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: created 2024-04-26T04:56:08.401096+0000 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: modified 2024-04-26T04:56:22.483807+0000 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: tableserver 0 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: root 0 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: session_timeout 60 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: session_autoclose 300 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: max_file_size 1099511627776 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: max_xattr_size 65536 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: required_client_features {} 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: last_failure 0 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: last_failure_osd_epoch 0 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:01:58.157 INFO:journalctl@ceph.mon.c.smithi152.stdout: max_mds 1 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: in 0 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: up {0=14640} 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: failed 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: damaged 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: stopped 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: data_pools [4] 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: metadata_pool 3 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: inline_data disabled 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: balancer 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: bal_rank_mask -1 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: standby_count_wanted 1 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: qdb_cluster leader: 0 members: 2024-04-26T05:01:58.158 INFO:journalctl@ceph.mon.c.smithi152.stdout: [mds.foo.smithi152.rnhpxy{0:14640} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout: Standby daemons: 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout: 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout: [mds.foo.smithi195.bqeurw{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???).osd e91 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 ceph-mon[106538]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 bash[106501]: 0bfb660a4a44fa6e6b8f02247942cb4c6942e91ca8289bf4ed5d03382c7df4d6 2024-04-26T05:01:58.159 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:01:57 smithi152 systemd[1]: Started Ceph mon.c for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:59.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:01:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:01:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:03.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:02:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:02:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:02:03.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:03 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/690564597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:03.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:03 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:02:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:03 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:02:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:03 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:02:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:03 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/690564597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:04.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:02:04] "GET /metrics HTTP/1.1" 200 33841 "" "Prometheus/2.43.0" 2024-04-26T05:02:04.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: mon.c calling monitor election 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: mon.b calling monitor election 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: mon.a calling monitor election 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: monmap epoch 3 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: last_changed 2024-04-26T04:50:48.501641+0000 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: min_mon_release 17 (quincy) 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: election_strategy: 1 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:02:04.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: mgrmap e38: y(active, since 91s), standbys: x 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[102000]: daemon grafana.a on smithi195 is in error state 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:02:04.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/690564597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: mon.c calling monitor election 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: mon.b calling monitor election 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: mon.a calling monitor election 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: monmap epoch 3 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: last_changed 2024-04-26T04:50:48.501641+0000 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: min_mon_release 17 (quincy) 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: election_strategy: 1 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:02:04.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: mgrmap e38: y(active, since 91s), standbys: x 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:02:04.384 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:04 smithi152 ceph-mon[106538]: daemon grafana.a on smithi195 is in error state 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: mon.c calling monitor election 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: mon.b calling monitor election 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: mon.a calling monitor election 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: monmap epoch 3 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: last_changed 2024-04-26T04:50:48.501641+0000 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: min_mon_release 17 (quincy) 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: election_strategy: 1 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:02:04.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: mgrmap e38: y(active, since 91s), standbys: x 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:02:04.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:04 smithi195 ceph-mon[31427]: daemon grafana.a on smithi195 is in error state 2024-04-26T05:02:05.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:05 smithi152 ceph-mon[102000]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:05.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:05 smithi152 ceph-mon[106538]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:05.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:05 smithi195 ceph-mon[31427]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:06.201 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T05:02:06.402 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:06 smithi152 ceph-mon[102000]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T05:02:06.402 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:06 smithi152 ceph-mon[106538]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T05:02:06.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:06 smithi195 ceph-mon[31427]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T05:02:07.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:02:07.346 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:07 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1305119418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:07.346 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:07 smithi152 ceph-mon[106538]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:07.346 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:07 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1305119418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:07.346 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:07 smithi152 ceph-mon[102000]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:07.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:07 smithi195 ceph-mon[31427]: from='client.? 172.21.15.152:0/1305119418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:07.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:07 smithi195 ceph-mon[31427]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:07.786 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:08.177 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:07 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:02:07.906+0000 7f7001d67700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-26T05:02:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:09.358 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:09 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:09.358 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:09 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:09.358 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:09 smithi152 ceph-mon[102000]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:09.359 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:09 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:09.359 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:09 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:09.359 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:09 smithi152 ceph-mon[106538]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:09.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:09 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:09.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:09 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:09.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:09 smithi195 ceph-mon[31427]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:09.831 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:02:09.831 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.a smithi152 *:9093,9094 running (3m) 1s ago 7m 29.0M - 0.25.0 c8568f914cd2 252d0e292dd9 2024-04-26T05:02:09.831 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.a smithi195 *:3000 error 22s ago 6m - - 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi152.rnhpxy smithi152 running (5m) 1s ago 5m 16.4M - 17.2.7-902-gc61472c8 0de407c1b3af 6f0b0b86b7ef 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi195.bqeurw smithi195 running (5m) 22s ago 5m 15.4M - 17.2.7-902-gc61472c8 0de407c1b3af db58137dcdb3 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.x smithi195 *:8443,9283,8765 running (2m) 22s ago 11m 449M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 178e4f8e339a 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.y smithi152 *:8443,9283,8765 running (2m) 1s ago 12m 499M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 ec2ad8342f2f 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:mon.a smithi152 running (107s) 1s ago 12m 43.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 1b4ec8125acd 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:mon.b smithi195 running (11m) 22s ago 11m 51.6M 2048M 17.2.7-902-gc61472c8 0de407c1b3af bf477f6a2b3d 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:mon.c smithi152 running (12s) 1s ago 11m 18.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0bfb660a4a44 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.a smithi152 *:9100 running (2m) 1s ago 6m 15.0M - 1.5.0 0da6a335fe13 8d4a4fd1dbae 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.b smithi195 *:9100 running (2m) 22s ago 6m 15.7M - 1.5.0 0da6a335fe13 bdec586e73b3 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (10m) 1s ago 10m 49.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a6ef80972333 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (10m) 1s ago 10m 48.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 827c6f69d0b5 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (9m) 1s ago 9m 48.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3b4441be6ba1 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi152 running (9m) 1s ago 9m 46.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 1b39c722ab3d 2024-04-26T05:02:09.832 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi195 running (9m) 22s ago 9m 47.8M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 958de1ceb551 2024-04-26T05:02:09.833 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi195 running (8m) 22s ago 8m 46.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5a97e623bdc9 2024-04-26T05:02:09.833 INFO:teuthology.orchestra.run.smithi152.stdout:osd.6 smithi195 running (8m) 22s ago 8m 44.9M 2480M 17.2.7-902-gc61472c8 0de407c1b3af f77aacbd5fd5 2024-04-26T05:02:09.833 INFO:teuthology.orchestra.run.smithi152.stdout:osd.7 smithi195 running (7m) 22s ago 7m 52.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1529e54096a 2024-04-26T05:02:09.833 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.a smithi195 *:9095 running (2m) 22s ago 7m 53.4M - 2.43.0 a07b618ecd1d 153c7fab44cf 2024-04-26T05:02:10.432 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:10 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:10.432 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:10 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:10.433 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:10.087Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", public_addr=\"172.21.15.152\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.152\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-26T05:02:10.472 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:10 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:10.473 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:10 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:10.473 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:10 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:10.473 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:10 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:11.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T05:02:11.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:11 smithi195 ceph-mon[31427]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:11.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:11 smithi195 ceph-mon[31427]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:11.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:11 smithi152 ceph-mon[102000]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:11.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:11 smithi152 ceph-mon[102000]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:11.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:11 smithi152 ceph-mon[106538]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:11.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:11 smithi152 ceph-mon[106538]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:12.398 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:13.367 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:13 smithi152 ceph-mon[106538]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:13.367 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:13 smithi152 ceph-mon[102000]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:13.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:13 smithi195 ceph-mon[31427]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:02:14.468 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:02:14] "GET /metrics HTTP/1.1" 200 33841 "" "Prometheus/2.43.0" 2024-04-26T05:02:15.005 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.005 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.005 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:15.005 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[102000]: Upgrade: It appears safe to stop mon.b 2024-04-26T05:02:15.006 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T05:02:15.007 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:14 smithi152 ceph-mon[106538]: Upgrade: It appears safe to stop mon.b 2024-04-26T05:02:15.202 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.202 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.202 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:15.202 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:15.202 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-04-26T05:02:15.203 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:15.203 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:15.203 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:15.203 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:15.203 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:02:15.203 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T05:02:15.203 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:14 smithi195 ceph-mon[31427]: Upgrade: It appears safe to stop mon.b 2024-04-26T05:02:15.243 INFO:teuthology.orchestra.run.smithi152.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:15.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T05:02:16.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[102000]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:16.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[102000]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:16.242 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:16.242 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:16.523 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:16.523 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:16.523 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:16.523 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:16.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[106538]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:16.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[106538]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:16.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:16.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:16.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:16.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:16.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:16 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:16 smithi195 ceph-mon[31427]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:16 smithi195 ceph-mon[31427]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:16 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:16 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:16.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:16 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:16.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:16 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:16.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:16 smithi195 ceph-mon[31427]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:17.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:17 smithi152 ceph-mon[102000]: Upgrade: Updating mon.b 2024-04-26T05:02:17.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:17 smithi152 ceph-mon[102000]: Deploying daemon mon.b on smithi195 2024-04-26T05:02:17.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:17 smithi152 ceph-mon[102000]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:17 smithi152 ceph-mon[106538]: Upgrade: Updating mon.b 2024-04-26T05:02:17.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:17 smithi152 ceph-mon[106538]: Deploying daemon mon.b on smithi195 2024-04-26T05:02:17.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:17 smithi152 ceph-mon[106538]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.516 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:17 smithi195 ceph-mon[31427]: Upgrade: Updating mon.b 2024-04-26T05:02:17.516 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:17 smithi195 ceph-mon[31427]: Deploying daemon mon.b on smithi195 2024-04-26T05:02:17.516 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:17 smithi195 ceph-mon[31427]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:17 smithi195 systemd[1]: Stopping Ceph mon.b for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:02:18.215 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b[31390]: 2024-04-26T05:02:17.790+0000 7fe5aa517700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:02:18.215 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b[31390]: 2024-04-26T05:02:17.790+0000 7fe5aa517700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-26T05:02:18.215 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:17 smithi195 podman[90026]: 2024-04-26 05:02:17.95464531 +0000 UTC m=+0.297627220 container died bf477f6a2b3d1e874af67aba1c9c4c6a7d68894a4fb86d2be3e57184c76a8dcf (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:02:18.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:18 smithi195 podman[90026]: 2024-04-26 05:02:18.313068575 +0000 UTC m=+0.656050485 container cleanup bf477f6a2b3d1e874af67aba1c9c4c6a7d68894a4fb86d2be3e57184c76a8dcf (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T05:02:18.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:18 smithi195 bash[90026]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b 2024-04-26T05:02:18.932 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:18 smithi195 podman[90045]: 2024-04-26 05:02:18.589809293 +0000 UTC m=+0.631646888 container remove bf477f6a2b3d1e874af67aba1c9c4c6a7d68894a4fb86d2be3e57184c76a8dcf (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:02:18.932 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:18 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-26T05:02:18.932 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:18 smithi195 systemd[1]: Stopped Ceph mon.b for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:02:18.932 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:18 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.b.service: Consumed 8.748s CPU time. 2024-04-26T05:02:18.932 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:19.270 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:18 smithi195 systemd[1]: Starting Ceph mon.b for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:02:19.270 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 podman[90183]: 2024-04-26 05:02:19.141059663 +0000 UTC m=+0.107992069 container create fc88dd94e9a698dacd0c40eb0c0e2205b4712eabe063507a38bd28360c879953 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb) 2024-04-26T05:02:19.270 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 podman[90183]: 2024-04-26 05:02:19.051726677 +0000 UTC m=+0.018659082 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 podman[90183]: 2024-04-26 05:02:19.343364551 +0000 UTC m=+0.310296958 container init fc88dd94e9a698dacd0c40eb0c0e2205b4712eabe063507a38bd28360c879953 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:02:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 podman[90183]: 2024-04-26 05:02:19.34926499 +0000 UTC m=+0.316197396 container start fc88dd94e9a698dacd0c40eb0c0e2205b4712eabe063507a38bd28360c879953 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T05:02:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T05:02:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 7 2024-04-26T05:02:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: pidfile_write: ignore empty --pid-file 2024-04-26T05:02:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: load: jerasure load: lrc 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: RocksDB version: 7.9.2 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Git sha 0 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: DB SUMMARY 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: DB Session ID: ULORN6VYJYXNWBLECS1B 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: CURRENT file: CURRENT 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 3662587 ; 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.error_if_exists: 0 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.create_if_missing: 0 2024-04-26T05:02:19.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.paranoid_checks: 1 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.env: 0x55841311ec60 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.info_log: 0x558414377aa0 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T05:02:19.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.statistics: (nil) 2024-04-26T05:02:19.784 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.use_fsync: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_log_file_size: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.allow_fallocate: 1 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.use_direct_reads: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T05:02:19.785 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.db_log_dir: 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.wal_dir: 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.write_buffer_manager: 0x55841434a820 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T05:02:19.786 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.unordered_write: 0 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T05:02:19.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.row_cache: None 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.wal_filter: None 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.two_write_queues: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.wal_compression: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.atomic_flush: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.log_readahead_size: 0 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T05:02:19.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_background_jobs: 2 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_background_compactions: -1 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_subcompactions: 1 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T05:02:19.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T05:02:19.790 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T05:02:19.790 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T05:02:19.790 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_open_files: -1 2024-04-26T05:02:19.790 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T05:02:19.790 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T05:02:19.790 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T05:02:19.790 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_background_flushes: -1 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Compression algorithms supported: 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kZSTD supported: 0 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kXpressCompression supported: 0 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kZlibCompression supported: 1 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kSnappyCompression supported: 1 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kLZ4Compression supported: 1 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: kBZip2Compression supported: 0 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T05:02:19.791 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.merge_operator: 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_filter: None 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558414377ba0) 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout: cache_index_and_filter_blocks: 1 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T05:02:19.792 INFO:journalctl@ceph.mon.b.smithi195.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: pin_top_level_index_and_filter: 1 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_type: 0 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_block_index_type: 0 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_shortening: 1 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: checksum: 4 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: no_block_cache: 0 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache: 0x55841437b090 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_name: BinnedLRUCache 2024-04-26T05:02:19.793 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_options: 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: capacity : 536870912 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: num_shard_bits : 4 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: strict_capacity_limit : 0 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: high_pri_pool_ratio: 0.000 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_cache_compressed: (nil) 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: persistent_cache: (nil) 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_size: 4096 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_size_deviation: 10 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_restart_interval: 16 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: index_block_restart_interval: 1 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: metadata_block_size: 4096 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: partition_filters: 0 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: use_delta_encoding: 1 2024-04-26T05:02:19.794 INFO:journalctl@ceph.mon.b.smithi195.stdout: filter_policy: bloomfilter 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: whole_key_filtering: 1 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: verify_compression: 0 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: read_amp_bytes_per_bit: 0 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: format_version: 5 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: enable_index_compression: 1 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: block_align: 0 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_auto_readahead_size: 262144 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: prepopulate_block_cache: 0 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: initial_auto_readahead_size: 8192 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T05:02:19.795 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression: NoCompression 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.num_levels: 7 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T05:02:19.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T05:02:19.797 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T05:02:19.798 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T05:02:19.799 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T05:02:19.800 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T05:02:19.800 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T05:02:19.800 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T05:02:19.800 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T05:02:19.800 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T05:02:19.800 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.table_properties_collectors: 2024-04-26T05:02:19.801 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.inplace_update_support: 0 2024-04-26T05:02:19.801 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T05:02:19.801 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T05:02:19.801 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T05:02:19.801 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T05:02:19.801 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.bloom_locality: 0 2024-04-26T05:02:19.802 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.max_successive_merges: 0 2024-04-26T05:02:19.802 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T05:02:19.802 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T05:02:19.802 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T05:02:19.802 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T05:02:19.802 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.ttl: 2592000 2024-04-26T05:02:19.803 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T05:02:19.803 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T05:02:19.803 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T05:02:19.803 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.enable_blob_files: false 2024-04-26T05:02:19.803 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.min_blob_size: 0 2024-04-26T05:02:19.803 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T05:02:19.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T05:02:19.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T05:02:19.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T05:02:19.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T05:02:19.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T05:02:19.804 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T05:02:19.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T05:02:19.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T05:02:19.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:02:19.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13062, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T05:02:19.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T05:02:19.805 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f103e535-089e-428d-a0bd-5476bbbc0df7 2024-04-26T05:02:19.806 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107739378908, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T05:02:19.806 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T05:02:19.806 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107739390107, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2265102, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13067, "largest_seqno": 14495, "table_properties": {"data_size": 2258698, "index_size": 3977, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 12389, "raw_average_key_size": 22, "raw_value_size": 2246630, "raw_average_value_size": 4069, "num_data_blocks": 183, "num_entries": 552, "num_filter_entries": 552, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107739, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f103e535-089e-428d-a0bd-5476bbbc0df7", "db_session_id": "ULORN6VYJYXNWBLECS1B", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T05:02:19.806 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107739390212, "job": 1, "event": "recovery_finished"} 2024-04-26T05:02:19.806 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T05:02:19.806 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:02:19.807 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T05:02:19.807 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55841446c000 2024-04-26T05:02:19.807 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: DB pointer 0x558414456000 2024-04-26T05:02:19.807 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T05:02:19.807 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T05:02:19.807 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** DB Stats ** 2024-04-26T05:02:19.808 INFO:journalctl@ceph.mon.b.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:02:19.808 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T05:02:19.808 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:02:19.808 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:02:19.808 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T05:02:19.808 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:02:19.809 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:02:19.809 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.809 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-04-26T05:02:19.809 INFO:journalctl@ceph.mon.b.smithi195.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:02:19.809 INFO:journalctl@ceph.mon.b.smithi195.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T05:02:19.809 INFO:journalctl@ceph.mon.b.smithi195.stdout: L0 1/0 2.16 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:02:19.810 INFO:journalctl@ceph.mon.b.smithi195.stdout: L6 1/0 7.82 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T05:02:19.810 INFO:journalctl@ceph.mon.b.smithi195.stdout: Sum 2/0 9.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:02:19.810 INFO:journalctl@ceph.mon.b.smithi195.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:02:19.810 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.810 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** Compaction Stats [default] ** 2024-04-26T05:02:19.810 INFO:journalctl@ceph.mon.b.smithi195.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:02:19.810 INFO:journalctl@ceph.mon.b.smithi195.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T05:02:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 492.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T05:02:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T05:02:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:02:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T05:02:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T05:02:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T05:02:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T05:02:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T05:02:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout: Cumulative compaction: 0.00 GB write, 158.09 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:02:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout: Interval compaction: 0.00 GB write, 158.09 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:02:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T05:02:19.813 INFO:journalctl@ceph.mon.b.smithi195.stdout: Block cache BinnedLRUCache@0x55841437b090#7 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2024-04-26T05:02:19.813 INFO:journalctl@ceph.mon.b.smithi195.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-26T05:02:19.813 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.813 INFO:journalctl@ceph.mon.b.smithi195.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T05:02:19.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: starting mon.b rank 1 at public addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] at bind addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:19.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???) e3 preinit fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???).mds e9 new map 2024-04-26T05:02:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???).mds e9 print_map 2024-04-26T05:02:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout: e9 2024-04-26T05:02:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T05:02:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:02:19.815 INFO:journalctl@ceph.mon.b.smithi195.stdout: legacy client fscid: 1 2024-04-26T05:02:19.815 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.815 INFO:journalctl@ceph.mon.b.smithi195.stdout: Filesystem 'foo' (1) 2024-04-26T05:02:19.815 INFO:journalctl@ceph.mon.b.smithi195.stdout: fs_name foo 2024-04-26T05:02:19.815 INFO:journalctl@ceph.mon.b.smithi195.stdout: epoch 8 2024-04-26T05:02:19.815 INFO:journalctl@ceph.mon.b.smithi195.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T05:02:19.816 INFO:journalctl@ceph.mon.b.smithi195.stdout: created 2024-04-26T04:56:08.401096+0000 2024-04-26T05:02:19.816 INFO:journalctl@ceph.mon.b.smithi195.stdout: modified 2024-04-26T04:56:22.483807+0000 2024-04-26T05:02:19.816 INFO:journalctl@ceph.mon.b.smithi195.stdout: tableserver 0 2024-04-26T05:02:19.816 INFO:journalctl@ceph.mon.b.smithi195.stdout: root 0 2024-04-26T05:02:19.816 INFO:journalctl@ceph.mon.b.smithi195.stdout: session_timeout 60 2024-04-26T05:02:19.816 INFO:journalctl@ceph.mon.b.smithi195.stdout: session_autoclose 300 2024-04-26T05:02:19.817 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_file_size 1099511627776 2024-04-26T05:02:19.817 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_xattr_size 65536 2024-04-26T05:02:19.817 INFO:journalctl@ceph.mon.b.smithi195.stdout: required_client_features {} 2024-04-26T05:02:19.817 INFO:journalctl@ceph.mon.b.smithi195.stdout: last_failure 0 2024-04-26T05:02:19.817 INFO:journalctl@ceph.mon.b.smithi195.stdout: last_failure_osd_epoch 0 2024-04-26T05:02:19.817 INFO:journalctl@ceph.mon.b.smithi195.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:02:19.818 INFO:journalctl@ceph.mon.b.smithi195.stdout: max_mds 1 2024-04-26T05:02:19.818 INFO:journalctl@ceph.mon.b.smithi195.stdout: in 0 2024-04-26T05:02:19.818 INFO:journalctl@ceph.mon.b.smithi195.stdout: up {0=14640} 2024-04-26T05:02:19.818 INFO:journalctl@ceph.mon.b.smithi195.stdout: failed 2024-04-26T05:02:19.818 INFO:journalctl@ceph.mon.b.smithi195.stdout: damaged 2024-04-26T05:02:19.818 INFO:journalctl@ceph.mon.b.smithi195.stdout: stopped 2024-04-26T05:02:19.819 INFO:journalctl@ceph.mon.b.smithi195.stdout: data_pools [4] 2024-04-26T05:02:19.819 INFO:journalctl@ceph.mon.b.smithi195.stdout: metadata_pool 3 2024-04-26T05:02:19.819 INFO:journalctl@ceph.mon.b.smithi195.stdout: inline_data disabled 2024-04-26T05:02:19.819 INFO:journalctl@ceph.mon.b.smithi195.stdout: balancer 2024-04-26T05:02:19.819 INFO:journalctl@ceph.mon.b.smithi195.stdout: bal_rank_mask -1 2024-04-26T05:02:19.819 INFO:journalctl@ceph.mon.b.smithi195.stdout: standby_count_wanted 1 2024-04-26T05:02:19.820 INFO:journalctl@ceph.mon.b.smithi195.stdout: qdb_cluster leader: 0 members: 2024-04-26T05:02:19.820 INFO:journalctl@ceph.mon.b.smithi195.stdout: [mds.foo.smithi152.rnhpxy{0:14640} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.152:6834/2106214948,v1:172.21.15.152:6835/2106214948] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:02:19.820 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.820 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.820 INFO:journalctl@ceph.mon.b.smithi195.stdout: Standby daemons: 2024-04-26T05:02:19.820 INFO:journalctl@ceph.mon.b.smithi195.stdout: 2024-04-26T05:02:19.821 INFO:journalctl@ceph.mon.b.smithi195.stdout: [mds.foo.smithi195.bqeurw{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.195:6832/1381138119,v1:172.21.15.195:6833/1381138119] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:02:19.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???).osd e91 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T05:02:19.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:02:19.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:02:19.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:02:19.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 ceph-mon[90222]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-04-26T05:02:19.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 bash[90183]: fc88dd94e9a698dacd0c40eb0c0e2205b4712eabe063507a38bd28360c879953 2024-04-26T05:02:19.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:19 smithi195 systemd[1]: Started Ceph mon.b for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:02:20.530 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:20.085Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", public_addr=\"172.21.15.152\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.152\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: mon.b calling monitor election 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: mon.c calling monitor election 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: mon.a calling monitor election 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: monmap epoch 4 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: last_changed 2024-04-26T05:02:19.660203+0000 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: min_mon_release 19 (squid) 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: election_strategy: 1 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: mgrmap e39: y(active, since 107s), standbys: x 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: daemon grafana.a on smithi195 is in error state 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:21.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:21.033 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:21.033 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:20 smithi195 ceph-mon[90222]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: mon.b calling monitor election 2024-04-26T05:02:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:02:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: mon.c calling monitor election 2024-04-26T05:02:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:02:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: mon.a calling monitor election 2024-04-26T05:02:21.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: monmap epoch 4 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: last_changed 2024-04-26T05:02:19.660203+0000 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: min_mon_release 19 (squid) 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: election_strategy: 1 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: mgrmap e39: y(active, since 107s), standbys: x 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:02:21.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: daemon grafana.a on smithi195 is in error state 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[102000]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: mon.b calling monitor election 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: mon.c calling monitor election 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: mon.a calling monitor election 2024-04-26T05:02:21.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: monmap epoch 4 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: fsid 164326a4-0388-11ef-bc93-c7b262605968 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: last_changed 2024-04-26T05:02:19.660203+0000 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: created 2024-04-26T04:49:13.193487+0000 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: min_mon_release 19 (squid) 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: election_strategy: 1 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: 0: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.a 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.b 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: 2: [v2:172.21.15.152:3301/0,v1:172.21.15.152:6790/0] mon.c 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: fsmap foo:1 {0=foo.smithi152.rnhpxy=up:active} 1 up:standby 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: mgrmap e39: y(active, since 107s), standbys: x 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:02:21.133 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:02:21.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: daemon grafana.a on smithi195 is in error state 2024-04-26T05:02:21.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:21.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:21.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:21.134 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:20 smithi152 ceph-mon[106538]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:23.260 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:22 smithi195 ceph-mon[90222]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:23.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:22 smithi152 ceph-mon[102000]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:23.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:22 smithi152 ceph-mon[106538]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:24.611 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:02:24] "GET /metrics HTTP/1.1" 200 33839 "" "Prometheus/2.43.0" 2024-04-26T05:02:24.611 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:02:24.398+0000 7f7001d67700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-26T05:02:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-mon[102000]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:24 smithi152 ceph-mon[106538]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:25.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:24 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:25.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:24 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:25.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:24 smithi195 ceph-mon[90222]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:25.989 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:25 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:25.990 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:25 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:26.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:25 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:26.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:25 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:26.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:25 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:26.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:25 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:27.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:26 smithi195 ceph-mon[90222]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:27.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:26 smithi152 ceph-mon[102000]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:27.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:26 smithi152 ceph-mon[106538]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: Reconfiguring mon.a (monmap changed)... 2024-04-26T05:02:29.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[102000]: Reconfiguring daemon mon.a on smithi152 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: Reconfiguring mon.a (monmap changed)... 2024-04-26T05:02:29.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:29.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:29.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:29.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:29 smithi152 ceph-mon[106538]: Reconfiguring daemon mon.a on smithi152 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: Reconfiguring mon.a (monmap changed)... 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:29.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:29.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:29 smithi195 ceph-mon[90222]: Reconfiguring daemon mon.a on smithi152 2024-04-26T05:02:31.278 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: Reconfiguring daemon mgr.y on smithi152 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[106538]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:31.279 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: Reconfiguring daemon mgr.y on smithi152 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:30 smithi152 ceph-mon[102000]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:31.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:31.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T05:02:31.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:02:31.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:02:31.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:31.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: Reconfiguring daemon mgr.y on smithi152 2024-04-26T05:02:31.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:30 smithi195 ceph-mon[90222]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: Reconfiguring mon.c (monmap changed)... 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: Reconfiguring daemon mon.c on smithi152 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:33.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:33.078 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: Reconfiguring mon.c (monmap changed)... 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: Reconfiguring daemon mon.c on smithi152 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: Reconfiguring mon.c (monmap changed)... 2024-04-26T05:02:33.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:33.080 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:33.080 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:33.080 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: Reconfiguring daemon mon.c on smithi152 2024-04-26T05:02:33.080 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:33.080 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:34.449 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:02:34] "GET /metrics HTTP/1.1" 200 33831 "" "Prometheus/2.43.0" 2024-04-26T05:02:34.802 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[106538]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.0 on smithi152 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[106538]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:34.803 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[102000]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T05:02:34.804 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:02:34.804 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:34.804 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.0 on smithi152 2024-04-26T05:02:34.804 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:34 smithi152 ceph-mon[102000]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:35.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:34 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:35.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:34 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:35.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:34 smithi195 ceph-mon[90222]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T05:02:35.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:34 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:02:35.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:34 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:35.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:34 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.0 on smithi152 2024-04-26T05:02:35.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:34 smithi195 ceph-mon[90222]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[102000]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T05:02:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:02:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.1 on smithi152 2024-04-26T05:02:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:36.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:36.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[106538]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T05:02:36.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:02:36.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:36.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:36 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.1 on smithi152 2024-04-26T05:02:36.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:36 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:36.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:36 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:36.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:36 smithi195 ceph-mon[90222]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T05:02:36.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:36 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:02:36.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:36 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:36.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:36 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.1 on smithi152 2024-04-26T05:02:37.470 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[102000]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:37.470 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:37.470 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:37.470 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:02:37.470 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:37.470 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[106538]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:37.470 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:37.471 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:37.471 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:02:37.471 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:37.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:37 smithi195 ceph-mon[90222]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:37.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:37.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:37.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:02:37.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:38 smithi152 ceph-mon[102000]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T05:02:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:38 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.2 on smithi152 2024-04-26T05:02:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:38 smithi152 ceph-mon[106538]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T05:02:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:38 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.2 on smithi152 2024-04-26T05:02:38.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:38 smithi195 ceph-mon[90222]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T05:02:38.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:38 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.2 on smithi152 2024-04-26T05:02:39.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:39.515 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[102000]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:39.515 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:39.515 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:39.515 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:02:39.515 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:39.515 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[106538]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:39.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:39.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:39.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:02:39.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:39.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:39 smithi195 ceph-mon[90222]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:39 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:39 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:39 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:02:39.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:39 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:40.277 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:40 smithi152 ceph-mon[102000]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T05:02:40.277 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:40 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.3 on smithi152 2024-04-26T05:02:40.278 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:40 smithi152 ceph-mon[106538]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T05:02:40.278 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:40 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.3 on smithi152 2024-04-26T05:02:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:40 smithi195 ceph-mon[90222]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T05:02:40.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:40 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.3 on smithi152 2024-04-26T05:02:41.624 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:41 smithi195 ceph-mon[90222]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:41.624 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:41.624 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:41.624 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:02:41.624 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[102000]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[106538]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:41.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.rnhpxy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:02:41.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:42.625 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:42 smithi195 ceph-mon[90222]: Reconfiguring mds.foo.smithi152.rnhpxy (monmap changed)... 2024-04-26T05:02:42.625 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:42 smithi195 ceph-mon[90222]: Reconfiguring daemon mds.foo.smithi152.rnhpxy on smithi152 2024-04-26T05:02:42.626 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:42 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:42.626 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:42 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:42.626 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:42 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:42.626 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:42 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:42.626 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:42 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[102000]: Reconfiguring mds.foo.smithi152.rnhpxy (monmap changed)... 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[102000]: Reconfiguring daemon mds.foo.smithi152.rnhpxy on smithi152 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[106538]: Reconfiguring mds.foo.smithi152.rnhpxy (monmap changed)... 2024-04-26T05:02:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[106538]: Reconfiguring daemon mds.foo.smithi152.rnhpxy on smithi152 2024-04-26T05:02:42.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:42.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:42.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:02:42.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:02:42.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:42 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:43.523 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:43 smithi195 ceph-mon[90222]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:43.523 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:43 smithi195 ceph-mon[90222]: Reconfiguring mon.b (monmap changed)... 2024-04-26T05:02:43.523 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:43 smithi195 ceph-mon[90222]: Reconfiguring daemon mon.b on smithi195 2024-04-26T05:02:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:43 smithi152 ceph-mon[102000]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:43 smithi152 ceph-mon[102000]: Reconfiguring mon.b (monmap changed)... 2024-04-26T05:02:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:43 smithi152 ceph-mon[102000]: Reconfiguring daemon mon.b on smithi195 2024-04-26T05:02:43.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:43 smithi152 ceph-mon[106538]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:43.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:43 smithi152 ceph-mon[106538]: Reconfiguring mon.b (monmap changed)... 2024-04-26T05:02:43.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:43 smithi152 ceph-mon[106538]: Reconfiguring daemon mon.b on smithi195 2024-04-26T05:02:44.575 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:02:44] "GET /metrics HTTP/1.1" 200 33831 "" "Prometheus/2.43.0" 2024-04-26T05:02:44.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T05:02:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:02:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:02:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: Reconfiguring daemon mgr.x on smithi195 2024-04-26T05:02:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:44 smithi195 ceph-mon[90222]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: Reconfiguring daemon mgr.x on smithi195 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[102000]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:44.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:44.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T05:02:44.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:02:44.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:02:44.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: Reconfiguring daemon mgr.x on smithi195 2024-04-26T05:02:44.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:44 smithi152 ceph-mon[106538]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:46 smithi195 ceph-mon[90222]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T05:02:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T05:02:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:46 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.4 on smithi195 2024-04-26T05:02:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[102000]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T05:02:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T05:02:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.4 on smithi195 2024-04-26T05:02:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[106538]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T05:02:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T05:02:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:46 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.4 on smithi195 2024-04-26T05:02:47.520 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:47 smithi195 ceph-mon[90222]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.521 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:47 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:47.521 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:47 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:47.521 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:47 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T05:02:47.521 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:47 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[102000]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T05:02:47.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:47.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[106538]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:47.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:47.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T05:02:47.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:47 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:48 smithi195 ceph-mon[90222]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T05:02:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:48 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.5 on smithi195 2024-04-26T05:02:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:48 smithi152 ceph-mon[102000]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T05:02:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:48 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.5 on smithi195 2024-04-26T05:02:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:48 smithi152 ceph-mon[106538]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T05:02:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:48 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.5 on smithi195 2024-04-26T05:02:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:49.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:49.371 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:49 smithi195 ceph-mon[90222]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:49.371 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:49 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:49.371 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:49 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:49.371 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:49 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T05:02:49.371 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:49 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:49.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[102000]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:49.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:49.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:49.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T05:02:49.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:49.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[106538]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:49.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:49.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:49.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T05:02:49.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:49 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:50 smithi195 ceph-mon[90222]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T05:02:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:50 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.6 on smithi195 2024-04-26T05:02:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T05:02:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[102000]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T05:02:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.6 on smithi195 2024-04-26T05:02:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T05:02:50.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:50.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[106538]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T05:02:50.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.6 on smithi195 2024-04-26T05:02:50.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:50.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:50.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T05:02:50.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:51 smithi195 ceph-mon[90222]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:51 smithi195 ceph-mon[90222]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T05:02:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:51 smithi195 ceph-mon[90222]: Reconfiguring daemon osd.7 on smithi195 2024-04-26T05:02:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:51 smithi152 ceph-mon[102000]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:51 smithi152 ceph-mon[102000]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T05:02:51.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:51 smithi152 ceph-mon[102000]: Reconfiguring daemon osd.7 on smithi195 2024-04-26T05:02:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:51 smithi152 ceph-mon[106538]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:51 smithi152 ceph-mon[106538]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T05:02:51.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:51 smithi152 ceph-mon[106538]: Reconfiguring daemon osd.7 on smithi195 2024-04-26T05:02:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[102000]: Reconfiguring mds.foo.smithi195.bqeurw (monmap changed)... 2024-04-26T05:02:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:02:53.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[102000]: Reconfiguring daemon mds.foo.smithi195.bqeurw on smithi195 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[102000]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[106538]: Reconfiguring mds.foo.smithi195.bqeurw (monmap changed)... 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[106538]: Reconfiguring daemon mds.foo.smithi195.bqeurw on smithi195 2024-04-26T05:02:53.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:52 smithi152 ceph-mon[106538]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:53.139 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:52 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:53.139 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:52 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:53.139 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:52 smithi195 ceph-mon[90222]: Reconfiguring mds.foo.smithi195.bqeurw (monmap changed)... 2024-04-26T05:02:53.139 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:52 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.bqeurw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:02:53.139 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:52 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:53.139 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:52 smithi195 ceph-mon[90222]: Reconfiguring daemon mds.foo.smithi195.bqeurw on smithi195 2024-04-26T05:02:53.139 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:52 smithi195 ceph-mon[90222]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.466 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:02:54] "GET /metrics HTTP/1.1" 200 33843 "" "Prometheus/2.43.0" 2024-04-26T05:02:54.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.787 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:54.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:54.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:54.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:54 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:54 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:02:56.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:56 smithi152 ceph-mon[102000]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:56.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:56 smithi152 ceph-mon[106538]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:57.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:56 smithi195 ceph-mon[90222]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:58.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:02:58 smithi152 ceph-mon[102000]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:58.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:02:58 smithi152 ceph-mon[106538]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:59.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:02:58 smithi195 ceph-mon[90222]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:59.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:02:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:02:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:03:00.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:00 smithi152 ceph-mon[102000]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:00.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:00 smithi152 ceph-mon[106538]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:01.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:00 smithi195 ceph-mon[90222]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:02.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:02 smithi152 ceph-mon[102000]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:02.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:02 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:02.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:02 smithi152 ceph-mon[106538]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:02.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:02 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:03.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:02 smithi195 ceph-mon[90222]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:03.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:02 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:04.576 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:03:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:03:04] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.43.0" 2024-04-26T05:03:04.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:04 smithi152 ceph-mon[102000]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:04.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:04 smithi152 ceph-mon[106538]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:05.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:04 smithi195 ceph-mon[90222]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:06.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:06 smithi152 ceph-mon[102000]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:06.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:06 smithi152 ceph-mon[106538]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:07.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:06 smithi195 ceph-mon[90222]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:08.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:08 smithi152 ceph-mon[102000]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:08.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:08 smithi152 ceph-mon[106538]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:09.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:08 smithi195 ceph-mon[90222]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:03:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:03:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:03:10.846 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:10 smithi195 ceph-mon[90222]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:10.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:10 smithi152 ceph-mon[102000]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:10.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:10 smithi152 ceph-mon[106538]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:12.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:12 smithi152 ceph-mon[102000]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:12.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:12 smithi152 ceph-mon[106538]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:13.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:12 smithi195 ceph-mon[90222]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:14.590 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:03:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:03:14] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.43.0" 2024-04-26T05:03:14.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:14 smithi152 ceph-mon[102000]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:14.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:14 smithi152 ceph-mon[106538]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:15.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:14 smithi195 ceph-mon[90222]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:16.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:16 smithi152 ceph-mon[102000]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:16.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:16 smithi152 ceph-mon[106538]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:17.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:16 smithi195 ceph-mon[90222]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:17.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:17 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:17.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:17 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:18.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:17 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:18.797 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:18 smithi152 ceph-mon[102000]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.797 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:18 smithi152 ceph-mon[106538]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T05:03:19.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:18 smithi195 ceph-mon[90222]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:19.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:03:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:03:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:03:19.114 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:20.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:20 smithi152 ceph-mon[102000]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:20.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:20 smithi152 ceph-mon[106538]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:20.978 INFO:teuthology.orchestra.run.smithi152.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:21.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:20 smithi195 ceph-mon[90222]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:21.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for all mons to upgrade' 2024-04-26T05:03:22.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:21 smithi195 ceph-mon[90222]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:22.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:21 smithi195 ceph-mon[90222]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:22.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:21 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:22.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:21 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[102000]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[102000]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[106538]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[106538]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:22.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:21 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:22.530 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:23.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:22 smithi195 ceph-mon[90222]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:23.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:22 smithi152 ceph-mon[102000]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:23.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:22 smithi152 ceph-mon[106538]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:24.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:03:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:03:24] "GET /metrics HTTP/1.1" 200 33847 "" "Prometheus/2.43.0" 2024-04-26T05:03:24.872 INFO:teuthology.orchestra.run.smithi152.stdout:wait for all mons to upgrade 2024-04-26T05:03:25.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:24 smithi152 ceph-mon[102000]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:25.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:24 smithi152 ceph-mon[106538]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:25.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:24 smithi195 ceph-mon[90222]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:25.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-26T05:03:26.213 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:27.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:26 smithi195 ceph-mon[90222]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:27.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:26 smithi152 ceph-mon[102000]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:27.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:26 smithi152 ceph-mon[106538]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.438 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T05:03:29.030 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:03:28 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:03:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:03:29.043 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:03:29.305 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:29.621 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:29.621 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.621 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.621 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/4157296127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:03:29.621 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/4157296127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.623 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/4157296127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:29.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[102000]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[102000]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[102000]: Upgrade: Setting container_image for all mon 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[102000]: Upgrade: Setting container_image for all crash 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[102000]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[102000]: Upgrade: osd.0 is safe to restart 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[106538]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.560 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[106538]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.561 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[106538]: Upgrade: Setting container_image for all mon 2024-04-26T05:03:30.561 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[106538]: Upgrade: Setting container_image for all crash 2024-04-26T05:03:30.561 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[106538]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:03:30.561 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:30 smithi152 ceph-mon[106538]: Upgrade: osd.0 is safe to restart 2024-04-26T05:03:30.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:30 smithi195 ceph-mon[90222]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:30 smithi195 ceph-mon[90222]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:30 smithi195 ceph-mon[90222]: Upgrade: Setting container_image for all mon 2024-04-26T05:03:30.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:30 smithi195 ceph-mon[90222]: Upgrade: Setting container_image for all crash 2024-04-26T05:03:30.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:30 smithi195 ceph-mon[90222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:03:30.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:30 smithi195 ceph-mon[90222]: Upgrade: osd.0 is safe to restart 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.a smithi152 *:9093,9094 running (4m) 3s ago 8m 27.8M - 0.25.0 c8568f914cd2 252d0e292dd9 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.a smithi195 *:3000 error 67s ago 8m - - 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi152.rnhpxy smithi152 running (7m) 3s ago 7m 16.8M - 17.2.7-902-gc61472c8 0de407c1b3af 6f0b0b86b7ef 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi195.bqeurw smithi195 running (7m) 67s ago 7m 15.6M - 17.2.7-902-gc61472c8 0de407c1b3af db58137dcdb3 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.x smithi195 *:8443,9283,8765 running (3m) 67s ago 12m 450M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 178e4f8e339a 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.y smithi152 *:8443,9283,8765 running (3m) 3s ago 14m 502M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 ec2ad8342f2f 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:mon.a smithi152 running (3m) 3s ago 14m 48.3M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 1b4ec8125acd 2024-04-26T05:03:31.687 INFO:teuthology.orchestra.run.smithi152.stdout:mon.b smithi195 running (72s) 67s ago 12m 20.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fc88dd94e9a6 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:mon.c smithi152 running (94s) 3s ago 12m 33.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0bfb660a4a44 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.a smithi152 *:9100 running (4m) 3s ago 7m 15.5M - 1.5.0 0da6a335fe13 8d4a4fd1dbae 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.b smithi195 *:9100 running (4m) 67s ago 7m 16.3M - 1.5.0 0da6a335fe13 bdec586e73b3 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (12m) 3s ago 12m 49.8M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a6ef80972333 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (11m) 3s ago 11m 48.9M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 827c6f69d0b5 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (11m) 3s ago 11m 48.7M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3b4441be6ba1 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi152 running (10m) 3s ago 10m 46.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 1b39c722ab3d 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi195 running (10m) 67s ago 10m 48.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 958de1ceb551 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi195 running (10m) 67s ago 10m 46.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5a97e623bdc9 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.6 smithi195 running (9m) 67s ago 9m 45.2M 2480M 17.2.7-902-gc61472c8 0de407c1b3af f77aacbd5fd5 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:osd.7 smithi195 running (9m) 67s ago 9m 52.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1529e54096a 2024-04-26T05:03:31.688 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.a smithi195 *:9095 running (3m) 67s ago 8m 59.7M - 2.43.0 a07b618ecd1d 153c7fab44cf 2024-04-26T05:03:32.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T05:03:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:32 smithi152 ceph-mon[106538]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:32 smithi152 ceph-mon[102000]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:32 smithi195 ceph-mon[90222]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.794 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:33.618 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[102000]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:33.618 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:33.618 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:33.619 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:03:33.619 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:33.619 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[106538]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:33.619 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:33.619 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:33.619 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:03:33.619 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:33.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:33 smithi195 ceph-mon[90222]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:33.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:33.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:33.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:03:33.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:34.220 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:03:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:03:34] "GET /metrics HTTP/1.1" 200 33843 "" "Prometheus/2.43.0" 2024-04-26T05:03:34.550 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:34 smithi152 ceph-mon[102000]: Upgrade: Updating osd.0 2024-04-26T05:03:34.550 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:34 smithi152 ceph-mon[102000]: Deploying daemon osd.0 on smithi152 2024-04-26T05:03:34.550 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:34 smithi152 ceph-mon[102000]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.551 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:34 smithi152 ceph-mon[106538]: Upgrade: Updating osd.0 2024-04-26T05:03:34.551 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:34 smithi152 ceph-mon[106538]: Deploying daemon osd.0 on smithi152 2024-04-26T05:03:34.551 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:34 smithi152 ceph-mon[106538]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:34 smithi195 ceph-mon[90222]: Upgrade: Updating osd.0 2024-04-26T05:03:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:34 smithi195 ceph-mon[90222]: Deploying daemon osd.0 on smithi152 2024-04-26T05:03:34.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:34 smithi195 ceph-mon[90222]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:35.310 INFO:teuthology.orchestra.run.smithi152.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:36.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T05:03:36.330 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[106538]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.330 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[106538]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:36.330 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[106538]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:36.331 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:36.331 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[102000]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.331 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[102000]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:36.331 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[102000]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:36.331 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:36 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:36.584 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:36 smithi195 ceph-mon[90222]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:36 smithi195 ceph-mon[90222]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:36 smithi195 ceph-mon[90222]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:36.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:36 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:37.880 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:37 smithi152 systemd[1]: Stopping Ceph osd.0 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:03:38.362 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:38 smithi152 ceph-mon[102000]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:38.362 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:38 smithi152 ceph-mon[106538]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:38.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:38 smithi195 ceph-mon[90222]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:39.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:03:38 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:03:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:03:39.323 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[43636]: 2024-04-26T05:03:39.036+0000 7fcd516cb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:03:39.323 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[43636]: 2024-04-26T05:03:39.036+0000 7fcd516cb700 -1 osd.0 91 *** Got signal Terminated *** 2024-04-26T05:03:39.323 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:39 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[43636]: 2024-04-26T05:03:39.036+0000 7fcd516cb700 -1 osd.0 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:03:39.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:39 smithi152 ceph-mon[102000]: osd.0 marked itself down and dead 2024-04-26T05:03:39.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:39 smithi152 ceph-mon[106538]: osd.0 marked itself down and dead 2024-04-26T05:03:39.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:39 smithi195 ceph-mon[90222]: osd.0 marked itself down and dead 2024-04-26T05:03:40.045 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:39 smithi152 podman[114424]: 2024-04-26 05:03:39.752782919 +0000 UTC m=+0.843831045 container died a6ef80972333e85baa51112d963b94faa7ef5fb3635058c3ef3a2032e865ee1d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-26T05:03:40.321 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:40 smithi152 podman[114424]: 2024-04-26 05:03:40.135650299 +0000 UTC m=+1.226698422 container cleanup a6ef80972333e85baa51112d963b94faa7ef5fb3635058c3ef3a2032e865ee1d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T05:03:40.321 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:40 smithi152 bash[114424]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0 2024-04-26T05:03:40.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:40 smithi152 ceph-mon[102000]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:40.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:40 smithi152 ceph-mon[102000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:03:40.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:40 smithi152 ceph-mon[102000]: osdmap e92: 8 total, 7 up, 8 in 2024-04-26T05:03:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:40 smithi152 ceph-mon[106538]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:40 smithi152 ceph-mon[106538]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:03:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:40 smithi152 ceph-mon[106538]: osdmap e92: 8 total, 7 up, 8 in 2024-04-26T05:03:40.631 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:40 smithi152 podman[114446]: 2024-04-26 05:03:40.412552684 +0000 UTC m=+0.650119359 container remove a6ef80972333e85baa51112d963b94faa7ef5fb3635058c3ef3a2032e865ee1d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T05:03:40.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:40 smithi195 ceph-mon[90222]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:40.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:40 smithi195 ceph-mon[90222]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:03:40.682 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:40 smithi195 ceph-mon[90222]: osdmap e92: 8 total, 7 up, 8 in 2024-04-26T05:03:41.083 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:40 smithi152 podman[114543]: 2024-04-26 05:03:40.661321152 +0000 UTC m=+0.018278130 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:41.084 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:40 smithi152 podman[114543]: 2024-04-26 05:03:40.763928908 +0000 UTC m=+0.120885874 container create 887b5cb4273d19e02b44c7dcd710e43e498b7237c1b67b9f309a23fe2860fa74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T05:03:41.084 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:40 smithi152 podman[114543]: 2024-04-26 05:03:40.974971443 +0000 UTC m=+0.331928408 container init 887b5cb4273d19e02b44c7dcd710e43e498b7237c1b67b9f309a23fe2860fa74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T05:03:41.084 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:40 smithi152 podman[114543]: 2024-04-26 05:03:40.980732943 +0000 UTC m=+0.337689908 container start 887b5cb4273d19e02b44c7dcd710e43e498b7237c1b67b9f309a23fe2860fa74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T05:03:41.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:41 smithi152 ceph-mon[102000]: osdmap e93: 8 total, 7 up, 8 in 2024-04-26T05:03:41.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:41 smithi152 ceph-mon[106538]: osdmap e93: 8 total, 7 up, 8 in 2024-04-26T05:03:41.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:41 smithi152 podman[114543]: 2024-04-26 05:03:41.083024175 +0000 UTC m=+0.439981149 container attach 887b5cb4273d19e02b44c7dcd710e43e498b7237c1b67b9f309a23fe2860fa74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-26T05:03:41.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:41 smithi152 podman[114543]: 2024-04-26 05:03:41.273103662 +0000 UTC m=+0.630060635 container died 887b5cb4273d19e02b44c7dcd710e43e498b7237c1b67b9f309a23fe2860fa74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T05:03:41.699 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:41 smithi195 ceph-mon[90222]: osdmap e93: 8 total, 7 up, 8 in 2024-04-26T05:03:42.332 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:41 smithi152 podman[114589]: 2024-04-26 05:03:41.912635883 +0000 UTC m=+0.634024134 container remove 887b5cb4273d19e02b44c7dcd710e43e498b7237c1b67b9f309a23fe2860fa74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:03:42.332 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:41 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-26T05:03:42.333 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:41 smithi152 systemd[1]: Stopped Ceph osd.0 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:03:42.333 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:41 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.0.service: Consumed 5.469s CPU time. 2024-04-26T05:03:42.333 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:42 smithi152 systemd[1]: Starting Ceph osd.0 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:03:42.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:42 smithi195 ceph-mon[90222]: pgmap v109: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:42 smithi152 ceph-mon[102000]: pgmap v109: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:42 smithi152 ceph-mon[106538]: pgmap v109: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.630 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:42 smithi152 podman[114698]: 2024-04-26 05:03:42.25214676 +0000 UTC m=+0.018064513 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:42.631 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:42 smithi152 podman[114698]: 2024-04-26 05:03:42.388887738 +0000 UTC m=+0.154805478 container create e0e6c0ae8c9f014d6bb17f6aed7063e688514ce8a6682114694f8338e7783979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T05:03:43.081 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:42 smithi152 podman[114698]: 2024-04-26 05:03:42.6338624 +0000 UTC m=+0.399780144 container init e0e6c0ae8c9f014d6bb17f6aed7063e688514ce8a6682114694f8338e7783979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-26T05:03:43.081 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:42 smithi152 podman[114698]: 2024-04-26 05:03:42.640332555 +0000 UTC m=+0.406250296 container start e0e6c0ae8c9f014d6bb17f6aed7063e688514ce8a6682114694f8338e7783979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T05:03:43.081 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:42 smithi152 podman[114698]: 2024-04-26 05:03:42.742344817 +0000 UTC m=+0.508262575 container attach e0e6c0ae8c9f014d6bb17f6aed7063e688514ce8a6682114694f8338e7783979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-26T05:03:43.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate[114713]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:03:43.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate[114713]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T05:03:43.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 bash[114698]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:03:43.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 bash[114698]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T05:03:43.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 bash[114698]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T05:03:43.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 bash[114698]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 bash[114698]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 bash[114698]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 bash[114698]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate[114713]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate[114713]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate[114713]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate[114713]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate[114713]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T05:03:43.381 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 podman[114698]: 2024-04-26 05:03:43.105187325 +0000 UTC m=+0.871105067 container died e0e6c0ae8c9f014d6bb17f6aed7063e688514ce8a6682114694f8338e7783979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T05:03:44.050 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 podman[114861]: 2024-04-26 05:03:43.677980329 +0000 UTC m=+0.567685951 container remove e0e6c0ae8c9f014d6bb17f6aed7063e688514ce8a6682114694f8338e7783979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T05:03:44.050 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 podman[114934]: 2024-04-26 05:03:43.792653922 +0000 UTC m=+0.018115292 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:44.050 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:43 smithi152 podman[114934]: 2024-04-26 05:03:43.92281403 +0000 UTC m=+0.148275419 container create 91fec4f30bc46101c87f38757cb12dad3643900a3b9982d6f8ae3c27cd85b275 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:03:44.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:03:44] "GET /metrics HTTP/1.1" 200 33843 "" "Prometheus/2.43.0" 2024-04-26T05:03:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-mon[102000]: pgmap v110: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:44.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-mon[106538]: pgmap v110: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:44.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:44.631 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:44 smithi152 podman[114934]: 2024-04-26 05:03:44.133364946 +0000 UTC m=+0.358826293 container init 91fec4f30bc46101c87f38757cb12dad3643900a3b9982d6f8ae3c27cd85b275 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-26T05:03:44.631 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:44 smithi152 podman[114934]: 2024-04-26 05:03:44.138836989 +0000 UTC m=+0.364298337 container start 91fec4f30bc46101c87f38757cb12dad3643900a3b9982d6f8ae3c27cd85b275 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:03:44.631 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-osd[114972]: -- 172.21.15.152:0/2568862984 <== mon.0 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55df8d65b680 con 0x55df8d625400 2024-04-26T05:03:44.631 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:44 smithi152 bash[114934]: 91fec4f30bc46101c87f38757cb12dad3643900a3b9982d6f8ae3c27cd85b275 2024-04-26T05:03:44.631 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:44 smithi152 systemd[1]: Started Ceph osd.0 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:03:44.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:44 smithi195 ceph-mon[90222]: pgmap v110: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:44.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:44.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:45.130 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[114948]: 2024-04-26T05:03:44.689+0000 7f179b5c37c0 -1 Falling back to public interface 2024-04-26T05:03:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:45 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:45.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:45 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:45.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:45 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:46 smithi152 ceph-mon[102000]: pgmap v111: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:46.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:46 smithi152 ceph-mon[102000]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:46.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:46 smithi152 ceph-mon[106538]: pgmap v111: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:46.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:46 smithi152 ceph-mon[106538]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:46.763 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:46 smithi195 ceph-mon[90222]: pgmap v111: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:46.763 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:46 smithi195 ceph-mon[90222]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:47.751 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[114948]: 2024-04-26T05:03:47.419+0000 7f179b5c37c0 -1 osd.0 91 log_to_monitors true 2024-04-26T05:03:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: pgmap v112: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:03:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:03:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.631 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: pgmap v112: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:48.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:48.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:03:48.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:03:48.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.632 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: pgmap v112: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:48.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.660 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:48.661 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:03:48.661 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:03:48.661 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.661 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.661 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:48.661 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:49.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:03:48 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:03:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:03:49.129 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:03:48 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[114948]: 2024-04-26T05:03:48.858+0000 7f1788a13700 -1 osd.0 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:03:49.745 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T05:03:49.745 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T05:03:49.745 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: from='osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:03:49.745 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:03:49.745 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: from='osd.0 ' entity='osd.0' 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[106538]: pgmap v114: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: from='osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: from='osd.0 ' entity='osd.0' 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:49.746 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:49 smithi152 ceph-mon[102000]: pgmap v114: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: from='osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: from='osd.0 ' entity='osd.0' 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:49 smithi195 ceph-mon[90222]: pgmap v114: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:50.759 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[102000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:03:50.759 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[102000]: osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967] boot 2024-04-26T05:03:50.759 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[102000]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T05:03:50.759 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:03:50.760 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[106538]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:03:50.760 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[106538]: osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967] boot 2024-04-26T05:03:50.760 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[106538]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T05:03:50.760 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:03:50.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:50 smithi195 ceph-mon[90222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:03:50.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:50 smithi195 ceph-mon[90222]: osd.0 [v2:172.21.15.152:6802/309583967,v1:172.21.15.152:6803/309583967] boot 2024-04-26T05:03:50.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:50 smithi195 ceph-mon[90222]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T05:03:50.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:03:52.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:51 smithi152 ceph-mon[102000]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T05:03:52.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:51 smithi152 ceph-mon[102000]: pgmap v117: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:52.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:51 smithi152 ceph-mon[106538]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T05:03:52.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:51 smithi152 ceph-mon[106538]: pgmap v117: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:51 smithi195 ceph-mon[90222]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T05:03:52.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:51 smithi195 ceph-mon[90222]: pgmap v117: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: pgmap v118: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: pgmap v119: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: pgmap v120: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: pgmap v121: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.032 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: pgmap v118: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: pgmap v119: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: pgmap v120: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: pgmap v121: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: pgmap v118: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: pgmap v119: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: pgmap v120: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: pgmap v121: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-26T05:03:54.132 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:03:54.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:03:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:03:54] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2024-04-26T05:03:55.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:54 smithi195 ceph-mon[90222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded) 2024-04-26T05:03:55.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:54 smithi152 ceph-mon[102000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded) 2024-04-26T05:03:55.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:54 smithi152 ceph-mon[106538]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded) 2024-04-26T05:03:56.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:55 smithi195 ceph-mon[90222]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:55 smithi152 ceph-mon[102000]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:55 smithi152 ceph-mon[106538]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:58.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:57 smithi195 ceph-mon[90222]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:58.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:57 smithi152 ceph-mon[102000]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:57 smithi152 ceph-mon[106538]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:59.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:03:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:03:58.659Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-26T05:03:59.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:03:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:03:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:04:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:03:59 smithi195 ceph-mon[90222]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:03:59 smithi152 ceph-mon[102000]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:03:59 smithi152 ceph-mon[106538]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:02.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:01 smithi195 ceph-mon[90222]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:01 smithi152 ceph-mon[102000]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:01 smithi152 ceph-mon[106538]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:03.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:03 smithi195 ceph-mon[90222]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:03.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:03.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:03.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:03.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[102000]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:03 smithi152 ceph-mon[106538]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:04.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:04:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:04:04] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-26T05:04:06.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:05 smithi152 ceph-mon[102000]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:06.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:05 smithi152 ceph-mon[106538]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:06.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:05 smithi195 ceph-mon[90222]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:07 smithi152 ceph-mon[102000]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:08.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:07 smithi152 ceph-mon[106538]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:08.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:07 smithi195 ceph-mon[90222]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:09.031 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:04:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:04:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.152\", device_class=\"ssd\", hostname=\"smithi152\", instance=\"smithi195.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.152\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:04:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:09 smithi152 ceph-mon[102000]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:09 smithi152 ceph-mon[106538]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:10.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:09 smithi195 ceph-mon[90222]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:12.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:11 smithi152 ceph-mon[102000]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:12.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:11 smithi152 ceph-mon[106538]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:12.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:11 smithi195 ceph-mon[90222]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:14.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:13 smithi152 ceph-mon[102000]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:14.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:13 smithi152 ceph-mon[106538]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:14.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:13 smithi195 ceph-mon[90222]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:14.629 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:04:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:04:14] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-26T05:04:16.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:15 smithi152 ceph-mon[102000]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:16.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:15 smithi152 ceph-mon[106538]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:16.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:15 smithi195 ceph-mon[90222]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:17 smithi195 ceph-mon[90222]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:18.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:17 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:18.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:17 smithi152 ceph-mon[102000]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:18.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:17 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:18.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:17 smithi152 ceph-mon[106538]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:18.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:17 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:20.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:19 smithi195 ceph-mon[90222]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:20.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:19 smithi152 ceph-mon[102000]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:20.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:19 smithi152 ceph-mon[106538]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:22.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:21 smithi195 ceph-mon[90222]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:22.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:21 smithi152 ceph-mon[102000]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:22.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:21 smithi152 ceph-mon[106538]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.171 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:23 smithi152 ceph-mon[102000]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.171 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:23 smithi152 ceph-mon[106538]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:23 smithi195 ceph-mon[90222]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:04:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:04:24] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-26T05:04:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:25 smithi195 ceph-mon[90222]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:26.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:25 smithi152 ceph-mon[102000]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:26.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:25 smithi152 ceph-mon[106538]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:28.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:27 smithi195 ceph-mon[90222]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:28.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:27 smithi152 ceph-mon[102000]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:28.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:27 smithi152 ceph-mon[106538]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:30.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:29 smithi195 ceph-mon[90222]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:29 smithi152 ceph-mon[102000]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:29 smithi152 ceph-mon[106538]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:32.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:31 smithi195 ceph-mon[90222]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:31 smithi152 ceph-mon[102000]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:32.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:31 smithi152 ceph-mon[106538]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:33.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:33.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:34.171 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:33 smithi152 ceph-mon[102000]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:34.172 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:33 smithi152 ceph-mon[106538]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:34.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:33 smithi195 ceph-mon[90222]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:34.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:04:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:04:34] "GET /metrics HTTP/1.1" 200 33969 "" "Prometheus/2.43.0" 2024-04-26T05:04:36.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:35 smithi195 ceph-mon[90222]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:35 smithi152 ceph-mon[102000]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:35 smithi152 ceph-mon[106538]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:37 smithi195 ceph-mon[90222]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:37 smithi152 ceph-mon[102000]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:38.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:37 smithi152 ceph-mon[106538]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:39.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T05:04:39.741 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:40.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:39 smithi152 ceph-mon[102000]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:40.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:39 smithi152 ceph-mon[106538]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:40.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:39 smithi195 ceph-mon[90222]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:41.591 INFO:teuthology.orchestra.run.smithi152.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.265 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.266 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:41 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:04:42.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:42.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:41 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:04:42.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for half of osds to upgrade' 2024-04-26T05:04:43.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[106538]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:43.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[106538]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:43.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[106538]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:43.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[106538]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:43.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[106538]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:04:43.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[106538]: Upgrade: osd.1 is safe to restart 2024-04-26T05:04:43.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[102000]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:43.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[102000]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:43.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[102000]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:43.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[102000]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:43.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[102000]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:04:43.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:42 smithi152 ceph-mon[102000]: Upgrade: osd.1 is safe to restart 2024-04-26T05:04:43.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:42 smithi195 ceph-mon[90222]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:43.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:42 smithi195 ceph-mon[90222]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:43.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:42 smithi195 ceph-mon[90222]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:43.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:42 smithi195 ceph-mon[90222]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:43.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:42 smithi195 ceph-mon[90222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:04:43.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:42 smithi195 ceph-mon[90222]: Upgrade: osd.1 is safe to restart 2024-04-26T05:04:43.563 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:44.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:04:44] "GET /metrics HTTP/1.1" 200 33969 "" "Prometheus/2.43.0" 2024-04-26T05:04:44.764 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[106538]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[106538]: Upgrade: Updating osd.1 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[106538]: Deploying daemon osd.1 on smithi152 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[102000]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[102000]: Upgrade: Updating osd.1 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:04:44.765 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:44.766 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:44 smithi152 ceph-mon[102000]: Deploying daemon osd.1 on smithi152 2024-04-26T05:04:45.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:44 smithi195 ceph-mon[90222]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:45.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:44 smithi195 ceph-mon[90222]: Upgrade: Updating osd.1 2024-04-26T05:04:45.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:45.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:04:45.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:45.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:44 smithi195 ceph-mon[90222]: Deploying daemon osd.1 on smithi152 2024-04-26T05:04:46.434 INFO:teuthology.orchestra.run.smithi152.stdout:wait for half of osds to upgrade 2024-04-26T05:04:46.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:46 smithi152 ceph-mon[102000]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:46.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:46 smithi152 ceph-mon[106538]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:46.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-26T05:04:47.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:46 smithi195 ceph-mon[90222]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:47.490 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:47.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:47 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:47.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:47 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:48.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:47 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:48.624 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:48 smithi152 ceph-mon[102000]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:48 smithi152 ceph-mon[106538]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:48.880 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:48 smithi152 systemd[1]: Stopping Ceph osd.1 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:04:49.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:48 smithi195 ceph-mon[90222]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:49.803 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:49 smithi152 ceph-mon[102000]: osd.1 marked itself down and dead 2024-04-26T05:04:49.804 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:49 smithi152 ceph-mon[106538]: osd.1 marked itself down and dead 2024-04-26T05:04:49.804 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[50026]: 2024-04-26T05:04:49.597+0000 7fa4c6573700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:04:49.804 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[50026]: 2024-04-26T05:04:49.597+0000 7fa4c6573700 -1 osd.1 96 *** Got signal Terminated *** 2024-04-26T05:04:49.804 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:49 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[50026]: 2024-04-26T05:04:49.597+0000 7fa4c6573700 -1 osd.1 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:04:50.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:49 smithi195 ceph-mon[90222]: osd.1 marked itself down and dead 2024-04-26T05:04:50.129 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:50 smithi152 podman[120257]: 2024-04-26 05:04:50.045371847 +0000 UTC m=+0.778050326 container died 827c6f69d0b551725d635a03e6c57c6eff832c82895fb8b400f69397dcc7ab08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:04:50.625 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:50 smithi152 podman[120257]: 2024-04-26 05:04:50.414059933 +0000 UTC m=+1.146738427 container cleanup 827c6f69d0b551725d635a03e6c57c6eff832c82895fb8b400f69397dcc7ab08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:04:50.625 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:50 smithi152 bash[120257]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1 2024-04-26T05:04:50.879 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:50 smithi152 podman[120332]: 2024-04-26 05:04:50.724291135 +0000 UTC m=+0.673370793 container remove 827c6f69d0b551725d635a03e6c57c6eff832c82895fb8b400f69397dcc7ab08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[102000]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[102000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[102000]: osdmap e97: 8 total, 7 up, 8 in 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1607198850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[106538]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[106538]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[106538]: osdmap e97: 8 total, 7 up, 8 in 2024-04-26T05:04:50.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:50 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1607198850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:51.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:50 smithi195 ceph-mon[90222]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:51.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:50 smithi195 ceph-mon[90222]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:51.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:50 smithi195 ceph-mon[90222]: osdmap e97: 8 total, 7 up, 8 in 2024-04-26T05:04:51.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:50 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1607198850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:51.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:51 smithi152 podman[120431]: 2024-04-26 05:04:50.979593087 +0000 UTC m=+0.018425395 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:51.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:51 smithi152 podman[120431]: 2024-04-26 05:04:51.092088428 +0000 UTC m=+0.130920728 container create 8d72542a0d8d068520742eb2592474b8c75cffa6d3c1f4415b9b8296174d018b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-26T05:04:51.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:51 smithi152 podman[120431]: 2024-04-26 05:04:51.292949709 +0000 UTC m=+0.331782001 container init 8d72542a0d8d068520742eb2592474b8c75cffa6d3c1f4415b9b8296174d018b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:04:51.381 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:51 smithi152 podman[120431]: 2024-04-26 05:04:51.298408386 +0000 UTC m=+0.337240679 container start 8d72542a0d8d068520742eb2592474b8c75cffa6d3c1f4415b9b8296174d018b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-26T05:04:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:51 smithi152 ceph-mon[102000]: osdmap e98: 8 total, 7 up, 8 in 2024-04-26T05:04:51.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:51 smithi152 ceph-mon[106538]: osdmap e98: 8 total, 7 up, 8 in 2024-04-26T05:04:51.880 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:51 smithi152 podman[120431]: 2024-04-26 05:04:51.386664495 +0000 UTC m=+0.425496788 container attach 8d72542a0d8d068520742eb2592474b8c75cffa6d3c1f4415b9b8296174d018b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T05:04:51.881 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:51 smithi152 podman[120431]: 2024-04-26 05:04:51.595069036 +0000 UTC m=+0.633901332 container died 8d72542a0d8d068520742eb2592474b8c75cffa6d3c1f4415b9b8296174d018b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T05:04:52.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:51 smithi195 ceph-mon[90222]: osdmap e98: 8 total, 7 up, 8 in 2024-04-26T05:04:52.584 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 podman[120478]: 2024-04-26 05:04:52.157219191 +0000 UTC m=+0.556327260 container remove 8d72542a0d8d068520742eb2592474b8c75cffa6d3c1f4415b9b8296174d018b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb) 2024-04-26T05:04:52.585 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-26T05:04:52.585 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 systemd[1]: Stopped Ceph osd.1 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:04:52.585 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.1.service: Consumed 6.482s CPU time. 2024-04-26T05:04:52.585 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 systemd[1]: Starting Ceph osd.1 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:04:52.854 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:52 smithi152 ceph-mon[102000]: pgmap v154: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:52.854 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:52 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3752400906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:52.855 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:52 smithi152 ceph-mon[106538]: pgmap v154: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:52.855 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:52 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3752400906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:52.855 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 podman[120613]: 2024-04-26 05:04:52.583823271 +0000 UTC m=+0.106623088 container create 38aae4a775e1237a511894ff37bcc2bd772ae99f822f30ef59e12e2d3b18f42f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:04:52.856 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 podman[120613]: 2024-04-26 05:04:52.495231625 +0000 UTC m=+0.018031457 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:53.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:52 smithi195 ceph-mon[90222]: pgmap v154: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:53.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:52 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3752400906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:53.130 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 podman[120613]: 2024-04-26 05:04:52.854039765 +0000 UTC m=+0.376839582 container init 38aae4a775e1237a511894ff37bcc2bd772ae99f822f30ef59e12e2d3b18f42f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:04:53.130 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 podman[120613]: 2024-04-26 05:04:52.859766394 +0000 UTC m=+0.382566219 container start 38aae4a775e1237a511894ff37bcc2bd772ae99f822f30ef59e12e2d3b18f42f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T05:04:53.130 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:52 smithi152 podman[120613]: 2024-04-26 05:04:52.962291142 +0000 UTC m=+0.485090964 container attach 38aae4a775e1237a511894ff37bcc2bd772ae99f822f30ef59e12e2d3b18f42f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T05:04:53.630 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate[120627]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:04:53.630 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate[120627]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T05:04:53.630 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 bash[120613]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:04:53.630 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 bash[120613]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 bash[120613]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 bash[120613]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 bash[120613]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 bash[120613]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 bash[120613]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate[120627]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate[120627]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate[120627]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate[120627]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate[120627]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T05:04:53.631 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:53 smithi152 podman[120613]: 2024-04-26 05:04:53.327049644 +0000 UTC m=+0.849849454 container died 38aae4a775e1237a511894ff37bcc2bd772ae99f822f30ef59e12e2d3b18f42f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:04:54.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:53 smithi195 ceph-mon[90222]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:54.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-mon[102000]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:54.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:53 smithi152 ceph-mon[106538]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:54.385 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:04:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:04:54] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-26T05:04:54.385 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 podman[120764]: 2024-04-26 05:04:54.133810571 +0000 UTC m=+0.795291841 container remove 38aae4a775e1237a511894ff37bcc2bd772ae99f822f30ef59e12e2d3b18f42f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T05:04:54.385 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 podman[120833]: 2024-04-26 05:04:54.254564601 +0000 UTC m=+0.018169692 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:54.650 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 podman[120833]: 2024-04-26 05:04:54.384933715 +0000 UTC m=+0.148538785 container create 5082cc3b9b9ce30f4ee2e0688dae663f5db3502cb1e0f3619715a444496b12d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T05:04:54.650 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 podman[120833]: 2024-04-26 05:04:54.621660435 +0000 UTC m=+0.385265515 container init 5082cc3b9b9ce30f4ee2e0688dae663f5db3502cb1e0f3619715a444496b12d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-26T05:04:54.651 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 podman[120833]: 2024-04-26 05:04:54.629625138 +0000 UTC m=+0.393230206 container start 5082cc3b9b9ce30f4ee2e0688dae663f5db3502cb1e0f3619715a444496b12d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-26T05:04:55.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:54 smithi195 ceph-mon[90222]: pgmap v155: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:55.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:54 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/505345959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:55.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:54 smithi152 ceph-mon[102000]: pgmap v155: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:55.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:54 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/505345959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:55.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:54 smithi152 ceph-mon[106538]: pgmap v155: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:55.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:54 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/505345959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:55.131 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 ceph-osd[120894]: -- 172.21.15.152:0/3177310443 <== mon.2 v2:172.21.15.152:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55607425d680 con 0x556074227400 2024-04-26T05:04:55.131 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 bash[120833]: 5082cc3b9b9ce30f4ee2e0688dae663f5db3502cb1e0f3619715a444496b12d5 2024-04-26T05:04:55.131 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:54 smithi152 systemd[1]: Started Ceph osd.1 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:04:55.630 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:55 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[120871]: 2024-04-26T05:04:55.181+0000 7f61fcf9a7c0 -1 Falling back to public interface 2024-04-26T05:04:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:56.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:56 smithi152 ceph-mon[102000]: pgmap v156: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:56.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:56 smithi152 ceph-mon[106538]: pgmap v156: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:57.257 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:56 smithi195 ceph-mon[90222]: pgmap v156: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:58.054 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:57 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1706900631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:58.054 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:57 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1706900631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:58.055 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:57 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[120871]: 2024-04-26T05:04:57.925+0000 7f61fcf9a7c0 -1 osd.1 96 log_to_monitors true 2024-04-26T05:04:58.186 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:57 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1706900631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[106538]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[106538]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[102000]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:59.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[102000]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T05:04:59.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:58 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:58 smithi195 ceph-mon[90222]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:04:59.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:58 smithi195 ceph-mon[90222]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T05:04:59.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:58 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:58 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:58 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:58 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:04:59.629 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[120871]: 2024-04-26T05:04:59.285+0000 7f61ea3ea700 -1 osd.1 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[102000]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[102000]: osdmap e99: 8 total, 7 up, 8 in 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[102000]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[102000]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/358357667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:00.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[106538]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T05:05:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[106538]: osdmap e99: 8 total, 7 up, 8 in 2024-04-26T05:05:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[106538]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[106538]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' 2024-04-26T05:05:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/358357667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:00.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:04:59 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:59 smithi195 ceph-mon[90222]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T05:05:00.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:59 smithi195 ceph-mon[90222]: osdmap e99: 8 total, 7 up, 8 in 2024-04-26T05:05:00.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:59 smithi195 ceph-mon[90222]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:00.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:59 smithi195 ceph-mon[90222]: from='osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538]' entity='osd.1' 2024-04-26T05:05:00.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:59 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/358357667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:00.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:59 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:00.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:04:59 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[106538]: pgmap v159: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[106538]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:05:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[106538]: osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538] boot 2024-04-26T05:05:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[106538]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T05:05:01.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:05:01.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[102000]: pgmap v159: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:01.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[102000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:05:01.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[102000]: osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538] boot 2024-04-26T05:05:01.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[102000]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T05:05:01.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:00 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:05:01.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:00 smithi195 ceph-mon[90222]: pgmap v159: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:01.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:00 smithi195 ceph-mon[90222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:05:01.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:00 smithi195 ceph-mon[90222]: osd.1 [v2:172.21.15.152:6810/994342538,v1:172.21.15.152:6811/994342538] boot 2024-04-26T05:05:01.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:00 smithi195 ceph-mon[90222]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T05:05:01.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:00 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:05:02.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:02 smithi152 ceph-mon[102000]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T05:05:02.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:02 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3190050091' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:02.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:02 smithi152 ceph-mon[106538]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T05:05:02.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:02 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3190050091' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:02.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:02 smithi195 ceph-mon[90222]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T05:05:02.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:02 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3190050091' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:03.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:05:03.315+0000 7f7001d67700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: pgmap v162: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: pgmap v162: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:03.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: pgmap v162: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:03.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:03.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:04.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:05:04] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.43.0" 2024-04-26T05:05:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[102000]: pgmap v163: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[102000]: pgmap v164: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[102000]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[102000]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2413267284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[106538]: pgmap v163: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[106538]: pgmap v164: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[106538]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[106538]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:04.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:04 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2413267284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:04.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:04 smithi195 ceph-mon[90222]: pgmap v163: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:04 smithi195 ceph-mon[90222]: pgmap v164: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-26T05:05:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:04 smithi195 ceph-mon[90222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:04 smithi195 ceph-mon[90222]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:04 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2413267284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:05 smithi195 ceph-mon[90222]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:05.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:05 smithi152 ceph-mon[102000]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:05.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:05 smithi152 ceph-mon[106538]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:06.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:06 smithi195 ceph-mon[90222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded) 2024-04-26T05:05:06.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:06 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2877462423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:06.781 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:06 smithi152 ceph-mon[102000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded) 2024-04-26T05:05:06.782 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:06 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2877462423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:06.782 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:06 smithi152 ceph-mon[106538]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded) 2024-04-26T05:05:06.782 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:06 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2877462423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:07.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:07 smithi195 ceph-mon[90222]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:07.797 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:07 smithi152 ceph-mon[106538]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:07.797 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:07 smithi152 ceph-mon[102000]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:09.020 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:08 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/662484980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:09.021 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:08 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/662484980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:09.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:08 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/662484980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:10.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:09 smithi195 ceph-mon[90222]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:09 smithi152 ceph-mon[102000]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:09 smithi152 ceph-mon[106538]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:11.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:11 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3600454920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:11.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:11 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3600454920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:11.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:11 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3600454920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:12.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:12 smithi152 ceph-mon[102000]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:12.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:12 smithi152 ceph-mon[106538]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:12.432 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:12 smithi195 ceph-mon[90222]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:13.703 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:13.703 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:13 smithi195 ceph-mon[90222]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:13.703 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:13 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/345348961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:13.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:13.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:13 smithi152 ceph-mon[106538]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:13.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:13 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/345348961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:13.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:13.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:13 smithi152 ceph-mon[102000]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:13.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:13 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/345348961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:14.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:05:14] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.43.0" 2024-04-26T05:05:16.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:16 smithi195 ceph-mon[90222]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:16.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:16 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1772146181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:16.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:16 smithi152 ceph-mon[102000]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:16.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:16 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1772146181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:16.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:16 smithi152 ceph-mon[106538]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:16.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:16 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1772146181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:18 smithi195 ceph-mon[90222]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:18 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2386164789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[102000]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2386164789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:18.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:18.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[106538]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:18.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:18.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:18.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2386164789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:18.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:19.625 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:19 smithi152 ceph-mon[102000]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:19.625 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:19 smithi152 ceph-mon[102000]: Upgrade: osd.2 is safe to restart 2024-04-26T05:05:19.626 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:19 smithi152 ceph-mon[106538]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:19.626 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:19 smithi152 ceph-mon[106538]: Upgrade: osd.2 is safe to restart 2024-04-26T05:05:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:19 smithi195 ceph-mon[90222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:05:19.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:19 smithi195 ceph-mon[90222]: Upgrade: osd.2 is safe to restart 2024-04-26T05:05:20.677 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[102000]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:20.677 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[102000]: Upgrade: Updating osd.2 2024-04-26T05:05:20.677 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:20.677 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:05:20.677 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:20.677 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[102000]: Deploying daemon osd.2 on smithi152 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/956030196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[106538]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[106538]: Upgrade: Updating osd.2 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[106538]: Deploying daemon osd.2 on smithi152 2024-04-26T05:05:20.678 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:20 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/956030196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:20.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:20 smithi195 ceph-mon[90222]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:20 smithi195 ceph-mon[90222]: Upgrade: Updating osd.2 2024-04-26T05:05:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:05:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:20 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:20 smithi195 ceph-mon[90222]: Deploying daemon osd.2 on smithi152 2024-04-26T05:05:20.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:20 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/956030196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:21.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:21 smithi152 ceph-mon[102000]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:21.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:21 smithi152 ceph-mon[106538]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:21 smithi195 ceph-mon[90222]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:22.993 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:22 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/719358983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:22.993 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:22 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/719358983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:23.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:22 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/719358983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:23.601 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:23 smithi152 systemd[1]: Stopping Ceph osd.2 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:05:23.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:23 smithi152 ceph-mon[106538]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:23.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:23 smithi152 ceph-mon[106538]: osd.2 marked itself down and dead 2024-04-26T05:05:23.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:23 smithi152 ceph-mon[102000]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:23.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:23 smithi152 ceph-mon[102000]: osd.2 marked itself down and dead 2024-04-26T05:05:23.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:23 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[56517]: 2024-04-26T05:05:23.599+0000 7fe1eb3ba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:05:23.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:23 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[56517]: 2024-04-26T05:05:23.599+0000 7fe1eb3ba700 -1 osd.2 101 *** Got signal Terminated *** 2024-04-26T05:05:23.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:23 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[56517]: 2024-04-26T05:05:23.599+0000 7fe1eb3ba700 -1 osd.2 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:05:24.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:23 smithi195 ceph-mon[90222]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:24.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:23 smithi195 ceph-mon[90222]: osd.2 marked itself down and dead 2024-04-26T05:05:24.482 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:05:24] "GET /metrics HTTP/1.1" 200 33962 "" "Prometheus/2.43.0" 2024-04-26T05:05:24.483 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:24 smithi152 podman[125569]: 2024-04-26 05:05:24.22259515 +0000 UTC m=+0.741822828 container died 3b4441be6ba1eb9710ffcc88fb089c1ef96a185a351b9cdebb523b85a071fccc (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-04-26T05:05:24.879 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:24 smithi152 ceph-mon[106538]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:05:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:24 smithi152 ceph-mon[106538]: osdmap e102: 8 total, 7 up, 8 in 2024-04-26T05:05:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:24 smithi152 ceph-mon[102000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:05:24.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:24 smithi152 ceph-mon[102000]: osdmap e102: 8 total, 7 up, 8 in 2024-04-26T05:05:24.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:24 smithi152 podman[125569]: 2024-04-26 05:05:24.58128078 +0000 UTC m=+1.100508459 container cleanup 3b4441be6ba1eb9710ffcc88fb089c1ef96a185a351b9cdebb523b85a071fccc (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:05:24.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:24 smithi152 bash[125569]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2 2024-04-26T05:05:25.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:24 smithi195 ceph-mon[90222]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:05:25.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:24 smithi195 ceph-mon[90222]: osdmap e102: 8 total, 7 up, 8 in 2024-04-26T05:05:25.215 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:24 smithi152 podman[125593]: 2024-04-26 05:05:24.89102684 +0000 UTC m=+0.663874100 container remove 3b4441be6ba1eb9710ffcc88fb089c1ef96a185a351b9cdebb523b85a071fccc (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:05:25.479 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:25 smithi152 podman[125717]: 2024-04-26 05:05:25.117756843 +0000 UTC m=+0.018260949 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:05:25.479 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:25 smithi152 podman[125717]: 2024-04-26 05:05:25.250568122 +0000 UTC m=+0.151072259 container create 6230459fd964c39731b0c7efe081bc45b2826f6e1c686bf05fdabcab555c7aeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:05:25.780 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:25 smithi152 ceph-mon[102000]: osdmap e103: 8 total, 7 up, 8 in 2024-04-26T05:05:25.780 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:25 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1699130375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:25.780 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:25 smithi152 ceph-mon[102000]: pgmap v177: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:25.780 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:25 smithi152 ceph-mon[106538]: osdmap e103: 8 total, 7 up, 8 in 2024-04-26T05:05:25.780 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:25 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1699130375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:25.781 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:25 smithi152 ceph-mon[106538]: pgmap v177: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:25.781 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:25 smithi152 podman[125717]: 2024-04-26 05:05:25.478961243 +0000 UTC m=+0.379465336 container init 6230459fd964c39731b0c7efe081bc45b2826f6e1c686bf05fdabcab555c7aeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T05:05:25.781 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:25 smithi152 podman[125717]: 2024-04-26 05:05:25.48477744 +0000 UTC m=+0.385281533 container start 6230459fd964c39731b0c7efe081bc45b2826f6e1c686bf05fdabcab555c7aeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T05:05:25.781 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:25 smithi152 podman[125717]: 2024-04-26 05:05:25.578758333 +0000 UTC m=+0.479262434 container attach 6230459fd964c39731b0c7efe081bc45b2826f6e1c686bf05fdabcab555c7aeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T05:05:26.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:25 smithi195 ceph-mon[90222]: osdmap e103: 8 total, 7 up, 8 in 2024-04-26T05:05:26.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:25 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1699130375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:26.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:25 smithi195 ceph-mon[90222]: pgmap v177: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:26.124 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:25 smithi152 podman[125717]: 2024-04-26 05:05:25.780040254 +0000 UTC m=+0.680544352 container died 6230459fd964c39731b0c7efe081bc45b2826f6e1c686bf05fdabcab555c7aeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:05:26.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:26 smithi152 podman[125763]: 2024-04-26 05:05:26.433101595 +0000 UTC m=+0.643968235 container remove 6230459fd964c39731b0c7efe081bc45b2826f6e1c686bf05fdabcab555c7aeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-26T05:05:26.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:26 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T05:05:26.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:26 smithi152 systemd[1]: Stopped Ceph osd.2 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:05:26.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:26 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.2.service: Consumed 5.833s CPU time. 2024-04-26T05:05:26.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:26 smithi152 systemd[1]: Starting Ceph osd.2 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:05:27.179 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:26 smithi152 podman[125871]: 2024-04-26 05:05:26.78261011 +0000 UTC m=+0.018739361 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:05:27.179 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:26 smithi152 podman[125871]: 2024-04-26 05:05:26.884143855 +0000 UTC m=+0.120273093 container create 1a28ec1bb6c4ab0f367c72e577bdc27dd10cfad7b1969706e3cb9aa133be01be (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:05:27.179 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 podman[125871]: 2024-04-26 05:05:27.0955172 +0000 UTC m=+0.331646439 container init 1a28ec1bb6c4ab0f367c72e577bdc27dd10cfad7b1969706e3cb9aa133be01be (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T05:05:27.179 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 podman[125871]: 2024-04-26 05:05:27.103498704 +0000 UTC m=+0.339627951 container start 1a28ec1bb6c4ab0f367c72e577bdc27dd10cfad7b1969706e3cb9aa133be01be (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-26T05:05:27.564 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 podman[125871]: 2024-04-26 05:05:27.178759943 +0000 UTC m=+0.414889191 container attach 1a28ec1bb6c4ab0f367c72e577bdc27dd10cfad7b1969706e3cb9aa133be01be (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb) 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate[125886]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 bash[125871]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 bash[125871]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 bash[125871]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 bash[125871]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 bash[125871]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 bash[125871]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:05:27.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 bash[125871]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T05:05:27.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate[125886]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T05:05:27.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate[125886]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T05:05:27.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate[125886]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T05:05:27.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate[125886]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T05:05:27.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate[125886]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:05:27.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate[125886]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T05:05:27.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:27 smithi152 podman[125871]: 2024-04-26 05:05:27.588411042 +0000 UTC m=+0.824540283 container died 1a28ec1bb6c4ab0f367c72e577bdc27dd10cfad7b1969706e3cb9aa133be01be (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-26T05:05:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:28 smithi195 ceph-mon[90222]: pgmap v178: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:28.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:28 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/774711271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:28.570 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:28 smithi152 ceph-mon[102000]: pgmap v178: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:28.571 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:28 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/774711271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:28.571 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:28 smithi152 ceph-mon[106538]: pgmap v178: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:28.571 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:28 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/774711271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:28.571 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 podman[126056]: 2024-04-26 05:05:28.350168948 +0000 UTC m=+0.756448123 container remove 1a28ec1bb6c4ab0f367c72e577bdc27dd10cfad7b1969706e3cb9aa133be01be (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T05:05:28.821 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 podman[126124]: 2024-04-26 05:05:28.473309268 +0000 UTC m=+0.018383353 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:05:28.821 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 podman[126124]: 2024-04-26 05:05:28.601750764 +0000 UTC m=+0.146824848 container create ec85e9f04bc66b15115beb0c708280552c1318b6221331710ce461d0e2e27761 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T05:05:29.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 podman[126124]: 2024-04-26 05:05:28.821092213 +0000 UTC m=+0.366166299 container init ec85e9f04bc66b15115beb0c708280552c1318b6221331710ce461d0e2e27761 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T05:05:29.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 podman[126124]: 2024-04-26 05:05:28.827576279 +0000 UTC m=+0.372650365 container start ec85e9f04bc66b15115beb0c708280552c1318b6221331710ce461d0e2e27761 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5) 2024-04-26T05:05:29.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 ceph-osd[126162]: -- 172.21.15.152:0/1801788727 <== mon.2 v2:172.21.15.152:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bf9f583680 con 0x55bf9f54cc00 2024-04-26T05:05:29.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 bash[126124]: ec85e9f04bc66b15115beb0c708280552c1318b6221331710ce461d0e2e27761 2024-04-26T05:05:29.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:28 smithi152 systemd[1]: Started Ceph osd.2 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:05:29.630 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[126138]: 2024-04-26T05:05:29.371+0000 7f033c45b7c0 -1 Falling back to public interface 2024-04-26T05:05:30.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:30.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:30.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:29 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:30.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:29 smithi195 ceph-mon[90222]: pgmap v179: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:30.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:29 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3269185170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[102000]: pgmap v179: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3269185170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:30.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[106538]: pgmap v179: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:30.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:29 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3269185170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:31.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:30 smithi152 ceph-mon[102000]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2024-04-26T05:05:31.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:30 smithi152 ceph-mon[106538]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2024-04-26T05:05:31.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:30 smithi195 ceph-mon[90222]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2024-04-26T05:05:32.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:31 smithi195 ceph-mon[90222]: pgmap v180: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:32.286 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:31 smithi152 ceph-mon[102000]: pgmap v180: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:32.286 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:31 smithi152 ceph-mon[106538]: pgmap v180: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:32.286 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[126138]: 2024-04-26T05:05:32.108+0000 7f033c45b7c0 -1 osd.2 101 log_to_monitors true 2024-04-26T05:05:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:32 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3349358575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:33.286 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:32 smithi195 ceph-mon[90222]: from='osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:05:33.286 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:32 smithi195 ceph-mon[90222]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:05:33.287 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.287 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:33.287 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.287 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3349358575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[102000]: from='osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:05:33.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[102000]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:05:33.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:33.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3349358575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:33.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[106538]: from='osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:05:33.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[106538]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:05:33.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:33.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:33.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:05:33 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[126138]: 2024-04-26T05:05:33.483+0000 7f03298ab700 -1 osd.2 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:05:34.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T05:05:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: osdmap e104: 8 total, 7 up, 8 in 2024-04-26T05:05:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: from='osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: pgmap v182: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: from='osd.2 ' entity='osd.2' 2024-04-26T05:05:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:34 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:05:34] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.43.0" 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: osdmap e104: 8 total, 7 up, 8 in 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: from='osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: pgmap v182: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: from='osd.2 ' entity='osd.2' 2024-04-26T05:05:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: osdmap e104: 8 total, 7 up, 8 in 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: from='osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: pgmap v182: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: from='osd.2 ' entity='osd.2' 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:34.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:34 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:35 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2955171727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:35 smithi195 ceph-mon[90222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:05:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:35 smithi195 ceph-mon[90222]: osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084] boot 2024-04-26T05:05:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:35 smithi195 ceph-mon[90222]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T05:05:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:35 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:05:35.354 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2955171727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:35.354 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[106538]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:05:35.354 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[106538]: osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084] boot 2024-04-26T05:05:35.354 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[106538]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T05:05:35.355 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:05:35.355 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2955171727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:35.355 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[102000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:05:35.355 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[102000]: osd.2 [v2:172.21.15.152:6818/2291823084,v1:172.21.15.152:6819/2291823084] boot 2024-04-26T05:05:35.355 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[102000]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T05:05:35.355 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:35 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:05:36.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:36 smithi195 ceph-mon[90222]: pgmap v184: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:36.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:36 smithi195 ceph-mon[90222]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T05:05:36.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:36 smithi152 ceph-mon[102000]: pgmap v184: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:36.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:36 smithi152 ceph-mon[102000]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T05:05:36.292 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:36 smithi152 ceph-mon[106538]: pgmap v184: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:36.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:36 smithi152 ceph-mon[106538]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T05:05:37.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:37 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3204400054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:37.296 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:37 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3204400054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:37.297 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:37 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3204400054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:38.017 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:37 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:05:37.943+0000 7f7001d67700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: pgmap v186: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: pgmap v186: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:38.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:38.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:38.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: pgmap v186: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:38.382 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:38.383 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:38 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:39.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:39 smithi195 ceph-mon[90222]: pgmap v187: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:39.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:39 smithi195 ceph-mon[90222]: pgmap v188: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:39.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:39 smithi195 ceph-mon[90222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:39.283 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:39 smithi195 ceph-mon[90222]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:39.283 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:39 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/599775713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[106538]: pgmap v187: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[106538]: pgmap v188: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[106538]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[106538]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/599775713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[102000]: pgmap v187: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[102000]: pgmap v188: 73 pgs: 4 active+undersized+degraded, 16 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-26T05:05:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[102000]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:39.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[102000]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T05:05:39.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:39 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/599775713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:40.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:40 smithi152 ceph-mon[102000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded) 2024-04-26T05:05:40.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:40 smithi152 ceph-mon[106538]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded) 2024-04-26T05:05:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:40 smithi195 ceph-mon[90222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded) 2024-04-26T05:05:41.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:41 smithi152 ceph-mon[102000]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:41.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:41 smithi152 ceph-mon[106538]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:41.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:41 smithi195 ceph-mon[90222]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:42 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2137542110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:42 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2137542110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:42.432 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:42 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2137542110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:43.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:43 smithi152 ceph-mon[102000]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:43.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:43 smithi152 ceph-mon[106538]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:43 smithi195 ceph-mon[90222]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:44 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3049947126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:44.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:44 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3049947126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:44.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:05:44] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.43.0" 2024-04-26T05:05:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:44 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3049947126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:45.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:45 smithi152 ceph-mon[102000]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:45.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:45 smithi152 ceph-mon[106538]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:45 smithi195 ceph-mon[90222]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:46 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2110444988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:46 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2110444988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:46.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:46 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2110444988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:47.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:47 smithi152 ceph-mon[102000]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:47.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:47 smithi152 ceph-mon[106538]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:47.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:47 smithi195 ceph-mon[90222]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:48.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:48.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:48.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:48.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:48 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3884548081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3884548081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:48.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:48 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3884548081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:49.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:49 smithi195 ceph-mon[90222]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:49.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:49 smithi152 ceph-mon[102000]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:49.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:49 smithi152 ceph-mon[106538]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:51 smithi195 ceph-mon[90222]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:51 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2153965723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:51.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:51 smithi152 ceph-mon[102000]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:51.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:51 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2153965723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:51.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:51 smithi152 ceph-mon[106538]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:51.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:51 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2153965723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:52.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:52 smithi195 ceph-mon[90222]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:52.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:52 smithi152 ceph-mon[102000]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:52.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:52 smithi152 ceph-mon[106538]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/640320756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[102000]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[102000]: Upgrade: osd.3 is safe to restart 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/640320756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[106538]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:53.719 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:53 smithi152 ceph-mon[106538]: Upgrade: osd.3 is safe to restart 2024-04-26T05:05:53.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:53 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/640320756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:53 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:53 smithi195 ceph-mon[90222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:05:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:53 smithi195 ceph-mon[90222]: Upgrade: osd.3 is safe to restart 2024-04-26T05:05:54.469 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:05:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:05:54] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.43.0" 2024-04-26T05:05:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:54 smithi195 ceph-mon[90222]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:54.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:54 smithi152 ceph-mon[102000]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:54.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:54 smithi152 ceph-mon[106538]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[102000]: Upgrade: Updating osd.3 2024-04-26T05:05:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:05:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[102000]: Deploying daemon osd.3 on smithi152 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/488800402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[106538]: Upgrade: Updating osd.3 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[106538]: Deploying daemon osd.3 on smithi152 2024-04-26T05:05:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:55 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/488800402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:56.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:55 smithi195 ceph-mon[90222]: Upgrade: Updating osd.3 2024-04-26T05:05:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:05:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:05:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:55 smithi195 ceph-mon[90222]: Deploying daemon osd.3 on smithi152 2024-04-26T05:05:56.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:55 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/488800402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:56.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:56 smithi152 ceph-mon[102000]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:56.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:56 smithi152 ceph-mon[106538]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:57.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:56 smithi195 ceph-mon[90222]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:57.986 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:57 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2876920964' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:57.986 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:57 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2876920964' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:58.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:57 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2876920964' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:05:58.240 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:57 smithi152 systemd[1]: Stopping Ceph osd.3 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:05:58.630 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:58 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[63099]: 2024-04-26T05:05:58.238+0000 7f79c778d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:05:58.630 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:58 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[63099]: 2024-04-26T05:05:58.238+0000 7f79c778d700 -1 osd.3 106 *** Got signal Terminated *** 2024-04-26T05:05:58.630 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:58 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[63099]: 2024-04-26T05:05:58.238+0000 7f79c778d700 -1 osd.3 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:05:59.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:58 smithi195 ceph-mon[90222]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:59.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:58 smithi195 ceph-mon[90222]: osd.3 marked itself down and dead 2024-04-26T05:05:59.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:58 smithi152 ceph-mon[102000]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:59.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:58 smithi152 ceph-mon[102000]: osd.3 marked itself down and dead 2024-04-26T05:05:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:58 smithi152 ceph-mon[106538]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:59.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:58 smithi152 ceph-mon[106538]: osd.3 marked itself down and dead 2024-04-26T05:05:59.497 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:59 smithi152 podman[130996]: 2024-04-26 05:05:59.178479646 +0000 UTC m=+1.047666173 container died 1b39c722ab3d36babf214381b4c86ecbe5f90f19286e5004706f27014060ab1c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T05:05:59.838 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:59 smithi152 ceph-mon[102000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:05:59.838 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:05:59 smithi152 ceph-mon[102000]: osdmap e107: 8 total, 7 up, 8 in 2024-04-26T05:05:59.839 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:59 smithi152 ceph-mon[106538]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:05:59.839 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:05:59 smithi152 ceph-mon[106538]: osdmap e107: 8 total, 7 up, 8 in 2024-04-26T05:05:59.839 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:59 smithi152 podman[130996]: 2024-04-26 05:05:59.594743399 +0000 UTC m=+1.463929918 container cleanup 1b39c722ab3d36babf214381b4c86ecbe5f90f19286e5004706f27014060ab1c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:05:59.839 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:59 smithi152 bash[130996]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3 2024-04-26T05:06:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:59 smithi195 ceph-mon[90222]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:06:00.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:05:59 smithi195 ceph-mon[90222]: osdmap e107: 8 total, 7 up, 8 in 2024-04-26T05:06:00.129 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:05:59 smithi152 podman[131018]: 2024-04-26 05:05:59.838224174 +0000 UTC m=+0.656116936 container remove 1b39c722ab3d36babf214381b4c86ecbe5f90f19286e5004706f27014060ab1c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2) 2024-04-26T05:06:00.551 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:00 smithi152 podman[131139]: 2024-04-26 05:06:00.264383718 +0000 UTC m=+0.103331350 container create e56561fecb41ef0423b8d9f68595c881548674882b7c8bf7ce0b62092a3da952 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:06:00.551 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:00 smithi152 podman[131139]: 2024-04-26 05:06:00.178646853 +0000 UTC m=+0.017594490 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:00.551 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:00 smithi152 podman[131139]: 2024-04-26 05:06:00.459456002 +0000 UTC m=+0.298403644 container init e56561fecb41ef0423b8d9f68595c881548674882b7c8bf7ce0b62092a3da952 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-26T05:06:00.551 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:00 smithi152 podman[131139]: 2024-04-26 05:06:00.465868685 +0000 UTC m=+0.304816317 container start e56561fecb41ef0423b8d9f68595c881548674882b7c8bf7ce0b62092a3da952 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T05:06:00.551 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:00 smithi152 podman[131139]: 2024-04-26 05:06:00.550568349 +0000 UTC m=+0.389515992 container attach e56561fecb41ef0423b8d9f68595c881548674882b7c8bf7ce0b62092a3da952 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T05:06:00.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:00 smithi152 ceph-mon[102000]: osdmap e108: 8 total, 7 up, 8 in 2024-04-26T05:06:00.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:00 smithi152 ceph-mon[102000]: pgmap v201: 73 pgs: 5 peering, 11 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:00.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:00 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3522262498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:00.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:00 smithi152 ceph-mon[106538]: osdmap e108: 8 total, 7 up, 8 in 2024-04-26T05:06:00.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:00 smithi152 ceph-mon[106538]: pgmap v201: 73 pgs: 5 peering, 11 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:00.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:00 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3522262498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:00.881 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:00 smithi152 podman[131139]: 2024-04-26 05:06:00.750766104 +0000 UTC m=+0.589713739 container died e56561fecb41ef0423b8d9f68595c881548674882b7c8bf7ce0b62092a3da952 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T05:06:01.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:00 smithi195 ceph-mon[90222]: osdmap e108: 8 total, 7 up, 8 in 2024-04-26T05:06:01.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:00 smithi195 ceph-mon[90222]: pgmap v201: 73 pgs: 5 peering, 11 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:01.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:00 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3522262498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:01.751 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:01 smithi152 podman[131184]: 2024-04-26 05:06:01.379445621 +0000 UTC m=+0.623253931 container remove e56561fecb41ef0423b8d9f68595c881548674882b7c8bf7ce0b62092a3da952 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T05:06:01.751 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:01 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T05:06:01.751 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:01 smithi152 systemd[1]: Stopped Ceph osd.3 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:06:01.752 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:01 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.3.service: Consumed 5.251s CPU time. 2024-04-26T05:06:01.752 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:01 smithi152 systemd[1]: Starting Ceph osd.3 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:06:02.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:01 smithi195 ceph-mon[90222]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-26T05:06:02.042 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:01 smithi152 ceph-mon[102000]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-26T05:06:02.042 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:01 smithi152 ceph-mon[106538]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-26T05:06:02.043 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:01 smithi152 podman[131295]: 2024-04-26 05:06:01.72652026 +0000 UTC m=+0.018250616 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:02.043 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:01 smithi152 podman[131295]: 2024-04-26 05:06:01.847567983 +0000 UTC m=+0.139298337 container create 2746e5bf8e37ada38a99737f2bc84f12b9b3765031991332d9c94447e13e321f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:06:02.043 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 podman[131295]: 2024-04-26 05:06:02.042079061 +0000 UTC m=+0.333809416 container init 2746e5bf8e37ada38a99737f2bc84f12b9b3765031991332d9c94447e13e321f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:06:02.380 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 podman[131295]: 2024-04-26 05:06:02.050280526 +0000 UTC m=+0.342010890 container start 2746e5bf8e37ada38a99737f2bc84f12b9b3765031991332d9c94447e13e321f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb) 2024-04-26T05:06:02.380 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 podman[131295]: 2024-04-26 05:06:02.1336304 +0000 UTC m=+0.425360767 container attach 2746e5bf8e37ada38a99737f2bc84f12b9b3765031991332d9c94447e13e321f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:06:02.758 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate[131310]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:06:02.759 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate[131310]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T05:06:02.759 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 bash[131295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 bash[131295]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 bash[131295]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 bash[131295]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 bash[131295]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 bash[131295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 bash[131295]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate[131310]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate[131310]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate[131310]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate[131310]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate[131310]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T05:06:02.760 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:02 smithi152 podman[131295]: 2024-04-26 05:06:02.526630794 +0000 UTC m=+0.818361140 container died 2746e5bf8e37ada38a99737f2bc84f12b9b3765031991332d9c94447e13e321f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T05:06:03.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:02 smithi195 ceph-mon[90222]: pgmap v202: 73 pgs: 5 peering, 11 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:03.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:02 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3262983049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:03.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:02 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:03.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:02 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[102000]: pgmap v202: 73 pgs: 5 peering, 11 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3262983049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[106538]: pgmap v202: 73 pgs: 5 peering, 11 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3262983049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:03.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:02 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:03.513 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 podman[131489]: 2024-04-26 05:06:03.163288059 +0000 UTC m=+0.631599239 container remove 2746e5bf8e37ada38a99737f2bc84f12b9b3765031991332d9c94447e13e321f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T05:06:03.513 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 podman[131559]: 2024-04-26 05:06:03.281340767 +0000 UTC m=+0.018542621 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:03.513 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 podman[131559]: 2024-04-26 05:06:03.380988088 +0000 UTC m=+0.118189985 container create 41cf15485f589acde56b3fde44cb5d2afc777b96fecf9831bcb8434c628ace9c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-26T05:06:04.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 podman[131559]: 2024-04-26 05:06:03.642274087 +0000 UTC m=+0.379475944 container init 41cf15485f589acde56b3fde44cb5d2afc777b96fecf9831bcb8434c628ace9c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424) 2024-04-26T05:06:04.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 podman[131559]: 2024-04-26 05:06:03.647838963 +0000 UTC m=+0.385040821 container start 41cf15485f589acde56b3fde44cb5d2afc777b96fecf9831bcb8434c628ace9c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T05:06:04.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 ceph-osd[131597]: -- 172.21.15.152:0/3309197470 <== mon.2 v2:172.21.15.152:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f0e47e3680 con 0x55f0e47ad000 2024-04-26T05:06:04.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 bash[131559]: 41cf15485f589acde56b3fde44cb5d2afc777b96fecf9831bcb8434c628ace9c 2024-04-26T05:06:04.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:03 smithi152 systemd[1]: Started Ceph osd.3 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:06:04.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:06:04] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.43.0" 2024-04-26T05:06:04.630 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:06:04.185+0000 7f1d514667c0 -1 Falling back to public interface 2024-04-26T05:06:04.921 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[102000]: pgmap v203: 73 pgs: 3 active+undersized+degraded, 5 peering, 23 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/740699038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[106538]: pgmap v203: 73 pgs: 3 active+undersized+degraded, 5 peering, 23 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:04.922 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:04 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/740699038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:05.030 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:04 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:04 smithi195 ceph-mon[90222]: pgmap v203: 73 pgs: 3 active+undersized+degraded, 5 peering, 23 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:04 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:04 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:05.031 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:04 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/740699038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:06.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:05 smithi152 ceph-mon[102000]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:06:06.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:05 smithi152 ceph-mon[106538]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:06:06.277 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:05 smithi195 ceph-mon[90222]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:06:07.118 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:06 smithi152 ceph-mon[102000]: pgmap v204: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:07.118 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:06 smithi152 ceph-mon[102000]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-26T05:06:07.118 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:06 smithi152 ceph-mon[106538]: pgmap v204: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:07.118 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:06 smithi152 ceph-mon[106538]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-26T05:06:07.119 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:06 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:06:06.941+0000 7f1d514667c0 -1 osd.3 106 log_to_monitors true 2024-04-26T05:06:07.147 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:06 smithi195 ceph-mon[90222]: pgmap v204: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:07.147 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:06 smithi195 ceph-mon[90222]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2605858528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[102000]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2605858528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:08.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[106538]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:06:08.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:07 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:07 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2605858528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:08.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:07 smithi195 ceph-mon[90222]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:06:08.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:07 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:07 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:07 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:07 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:08.629 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:06:08.275+0000 7f1d3e8b6700 -1 osd.3 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:06:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[106538]: pgmap v205: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[106538]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[106538]: osdmap e109: 8 total, 7 up, 8 in 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[106538]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[106538]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[102000]: pgmap v205: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[102000]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[102000]: osdmap e109: 8 total, 7 up, 8 in 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[102000]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:06:09.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[102000]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' 2024-04-26T05:06:09.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:09.132 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:08 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:09.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:08 smithi195 ceph-mon[90222]: pgmap v205: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:09.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:08 smithi195 ceph-mon[90222]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T05:06:09.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:08 smithi195 ceph-mon[90222]: osdmap e109: 8 total, 7 up, 8 in 2024-04-26T05:06:09.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:08 smithi195 ceph-mon[90222]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-26T05:06:09.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:08 smithi195 ceph-mon[90222]: from='osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714]' entity='osd.3' 2024-04-26T05:06:09.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:08 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:09.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:08 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2428775845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[102000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[102000]: osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714] boot 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[102000]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2428775845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[106538]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[106538]: osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714] boot 2024-04-26T05:06:10.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[106538]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T05:06:10.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:09 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:06:10.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:09 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2428775845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:10.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:09 smithi195 ceph-mon[90222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:06:10.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:09 smithi195 ceph-mon[90222]: osd.3 [v2:172.21.15.152:6826/3036260714,v1:172.21.15.152:6827/3036260714] boot 2024-04-26T05:06:10.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:09 smithi195 ceph-mon[90222]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T05:06:10.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:09 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:06:11.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:11 smithi195 ceph-mon[90222]: pgmap v208: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:11.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:11 smithi195 ceph-mon[90222]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T05:06:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T05:06:11.609 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:11 smithi152 ceph-mon[102000]: pgmap v208: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:11.609 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:11 smithi152 ceph-mon[102000]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T05:06:11.610 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:11 smithi152 ceph-mon[106538]: pgmap v208: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:11.610 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:11 smithi152 ceph-mon[106538]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T05:06:12.391 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:12 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1517278370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:12.391 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:12 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1517278370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:12.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T05:06:12.694 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:12 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1517278370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:06:13.007 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:13.129 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:06:12 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:06:12.973+0000 7f7001d67700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T05:06:13.403 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:13.403 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.403 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.404 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:13.405 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:13.405 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.405 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:13.405 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.405 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.405 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.405 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:13 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T05:06:13.636 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: pgmap v210: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:13.636 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.636 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:13.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:13 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T05:06:14.420 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[102000]: pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:14.420 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[102000]: pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:14.420 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[102000]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T05:06:14.421 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[102000]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T05:06:14.421 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:06:14] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.43.0" 2024-04-26T05:06:14.421 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[106538]: pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:14.421 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[106538]: pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:14.421 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[106538]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T05:06:14.421 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:14 smithi152 ceph-mon[106538]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T05:06:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:14 smithi195 ceph-mon[90222]: pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:14 smithi195 ceph-mon[90222]: pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-26T05:06:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:14 smithi195 ceph-mon[90222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T05:06:14.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:14 smithi195 ceph-mon[90222]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T05:06:14.862 INFO:teuthology.orchestra.run.smithi152.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:15.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:06:15.706 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:16.111 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[102000]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:16.111 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[102000]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:16.112 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:16.112 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[102000]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:16.112 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[106538]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:16.112 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[106538]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:16.112 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:16.112 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:15 smithi152 ceph-mon[106538]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:16.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:15 smithi195 ceph-mon[90222]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:16.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:15 smithi195 ceph-mon[90222]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:16.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:15 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:16.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:15 smithi195 ceph-mon[90222]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:17.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:16 smithi152 ceph-mon[102000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-04-26T05:06:17.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:16 smithi152 ceph-mon[106538]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-04-26T05:06:17.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:16 smithi195 ceph-mon[90222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-04-26T05:06:17.564 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.a smithi152 *:9093,9094 running (7m) 10s ago 11m 27.2M - 0.25.0 c8568f914cd2 252d0e292dd9 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.a smithi195 *:3000 error 9s ago 10m - - 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi152.rnhpxy smithi152 running (10m) 10s ago 9m 17.6M - 17.2.7-902-gc61472c8 0de407c1b3af 6f0b0b86b7ef 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi195.bqeurw smithi195 running (9m) 9s ago 9m 16.3M - 17.2.7-902-gc61472c8 0de407c1b3af db58137dcdb3 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.x smithi195 *:8443,9283,8765 running (6m) 9s ago 15m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 178e4f8e339a 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.y smithi152 *:8443,9283,8765 running (6m) 10s ago 16m 508M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 ec2ad8342f2f 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:mon.a smithi152 running (5m) 10s ago 17m 51.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 1b4ec8125acd 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:mon.b smithi195 running (3m) 9s ago 15m 35.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fc88dd94e9a6 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:mon.c smithi152 running (4m) 10s ago 15m 38.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0bfb660a4a44 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.a smithi152 *:9100 running (7m) 10s ago 10m 15.1M - 1.5.0 0da6a335fe13 8d4a4fd1dbae 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.b smithi195 *:9100 running (6m) 9s ago 10m 16.0M - 1.5.0 0da6a335fe13 bdec586e73b3 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (2m) 10s ago 14m 48.7M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 91fec4f30bc4 2024-04-26T05:06:17.565 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (83s) 10s ago 14m 47.7M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5082cc3b9b9c 2024-04-26T05:06:17.566 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (49s) 10s ago 14m 47.5M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 ec85e9f04bc6 2024-04-26T05:06:17.566 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi152 running (14s) 10s ago 13m 14.9M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 41cf15485f58 2024-04-26T05:06:17.566 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi195 running (13m) 9s ago 13m 51.8M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 958de1ceb551 2024-04-26T05:06:17.566 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi195 running (12m) 9s ago 12m 48.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5a97e623bdc9 2024-04-26T05:06:17.566 INFO:teuthology.orchestra.run.smithi152.stdout:osd.6 smithi195 running (12m) 9s ago 12m 48.9M 2480M 17.2.7-902-gc61472c8 0de407c1b3af f77aacbd5fd5 2024-04-26T05:06:17.566 INFO:teuthology.orchestra.run.smithi152.stdout:osd.7 smithi195 running (12m) 9s ago 12m 54.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1529e54096a 2024-04-26T05:06:17.566 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.a smithi195 *:9095 running (6m) 9s ago 11m 59.9M - 2.43.0 a07b618ecd1d 153c7fab44cf 2024-04-26T05:06:18.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:06:18.441 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:18.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:18 smithi152 ceph-mon[106538]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:18.506 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:18.506 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:18.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:18 smithi152 ceph-mon[102000]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:18.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:18.506 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:18 smithi195 ceph-mon[90222]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:19.616 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:19 smithi152 ceph-mon[102000]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:19.616 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:19 smithi152 ceph-mon[106538]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:19 smithi195 ceph-mon[90222]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:20.352 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.a smithi152 *:9093,9094 running (7m) 12s ago 11m 27.2M - 0.25.0 c8568f914cd2 252d0e292dd9 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.a smithi195 *:3000 error 12s ago 10m - - 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi152.rnhpxy smithi152 running (10m) 12s ago 10m 17.6M - 17.2.7-902-gc61472c8 0de407c1b3af 6f0b0b86b7ef 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:mds.foo.smithi195.bqeurw smithi195 running (10m) 12s ago 9m 16.3M - 17.2.7-902-gc61472c8 0de407c1b3af db58137dcdb3 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.x smithi195 *:8443,9283,8765 running (6m) 12s ago 15m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 178e4f8e339a 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.y smithi152 *:8443,9283,8765 running (6m) 12s ago 16m 508M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 ec2ad8342f2f 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:mon.a smithi152 running (5m) 12s ago 17m 51.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 1b4ec8125acd 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:mon.b smithi195 running (4m) 12s ago 15m 35.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fc88dd94e9a6 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:mon.c smithi152 running (4m) 12s ago 15m 38.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0bfb660a4a44 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.a smithi152 *:9100 running (7m) 12s ago 10m 15.1M - 1.5.0 0da6a335fe13 8d4a4fd1dbae 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.b smithi195 *:9100 running (6m) 12s ago 10m 16.0M - 1.5.0 0da6a335fe13 bdec586e73b3 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (2m) 12s ago 14m 48.7M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 91fec4f30bc4 2024-04-26T05:06:20.353 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (86s) 12s ago 14m 47.7M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5082cc3b9b9c 2024-04-26T05:06:20.354 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (51s) 12s ago 14m 47.5M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 ec85e9f04bc6 2024-04-26T05:06:20.354 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi152 running (17s) 12s ago 13m 14.9M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 41cf15485f58 2024-04-26T05:06:20.354 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi195 running (13m) 12s ago 13m 51.8M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 958de1ceb551 2024-04-26T05:06:20.354 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi195 running (12m) 12s ago 12m 48.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5a97e623bdc9 2024-04-26T05:06:20.354 INFO:teuthology.orchestra.run.smithi152.stdout:osd.6 smithi195 running (12m) 12s ago 12m 48.9M 2480M 17.2.7-902-gc61472c8 0de407c1b3af f77aacbd5fd5 2024-04-26T05:06:20.354 INFO:teuthology.orchestra.run.smithi152.stdout:osd.7 smithi195 running (12m) 12s ago 12m 54.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1529e54096a 2024-04-26T05:06:20.354 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.a smithi195 *:9095 running (6m) 12s ago 11m 59.9M - 2.43.0 a07b618ecd1d 153c7fab44cf 2024-04-26T05:06:20.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:20 smithi152 ceph-mon[102000]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:20 smithi152 ceph-mon[106538]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:20 smithi195 ceph-mon[90222]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 164326a4-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-26T05:06:21.226 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:21.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:21 smithi152 ceph-mon[102000]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:21.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:21 smithi152 ceph-mon[106538]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:21 smithi195 ceph-mon[90222]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:06:22.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:22 smithi152 ceph-mon[102000]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.589 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:22 smithi152 ceph-mon[106538]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:22 smithi195 ceph-mon[90222]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:23.164 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "mon": { 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "mgr": { 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "osd": { 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 4, 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "mds": { 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 2 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-26T05:06:23.165 INFO:teuthology.orchestra.run.smithi152.stdout: "overall": { 2024-04-26T05:06:23.166 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 6, 2024-04-26T05:06:23.166 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 9 2024-04-26T05:06:23.166 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-04-26T05:06:23.166 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-26T05:06:23.774 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:23 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:23.775 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:23 smithi152 ceph-mon[102000]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:23.775 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:23 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/632167880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:23.775 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:23 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:23.775 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:23 smithi152 ceph-mon[106538]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:23.775 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:23 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/632167880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:23.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:23 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:23.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:23 smithi195 ceph-mon[90222]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:23.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:23 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/632167880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:06:23.799 DEBUG:teuthology.parallel:result is None 2024-04-26T05:06:23.799 INFO:teuthology.run_tasks:Running task print... 2024-04-26T05:06:23.809 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-26T05:06:23.809 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T05:06:23.818 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T05:06:23.819 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T05:06:23.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T05:06:24.070 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:24.426 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:06:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:06:24] "GET /metrics HTTP/1.1" 200 34031 "" "Prometheus/2.43.0" 2024-04-26T05:06:25.938 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:06:26.248 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:25 smithi152 ceph-mon[106538]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:26.248 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:25 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2754633323' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:06:26.248 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:25 smithi152 ceph-mon[102000]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:26.248 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:25 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2754633323' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:06:26.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:25 smithi195 ceph-mon[90222]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:26.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:25 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2754633323' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:06:26.582 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":39,"flags":0,"active_gid":25898,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":473232309},{"type":"v1","addr":"172.21.15.152:6801","nonce":473232309}]},"active_addr":"172.21.15.152:6801/473232309","active_change":"2024-04-26T05:00:32.301063+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25594,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.152:8443/","prometheus":"http://172.21.15.152:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":91,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3417233726}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":2433553313}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":1915464197}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":1204358770}]}]} 2024-04-26T05:06:26.586 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T05:06:26.586 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T05:06:26.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:06:26.840 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:28.262 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:27 smithi152 ceph-mon[102000]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.262 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:27 smithi152 ceph-mon[106538]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:27 smithi195 ceph-mon[90222]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.699 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:06:28.699 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":111,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T05:06:10.273728+0000","last_up_change":"2024-04-26T05:06:09.269506+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:54:53.698027+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":86,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:56:07.054314+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:56:07.629218+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":95,"up_thru":110,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6803","nonce":309583967}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6805","nonce":309583967}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6809","nonce":309583967}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6807","nonce":309583967}]},"public_addr":"172.21.15.152:6803/309583967","cluster_addr":"172.21.15.152:6805/309583967","heartbeat_back_addr":"172.21.15.152:6809/309583967","heartbeat_front_addr":"172.21.15.152:6807/309583967","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":100,"up_thru":110,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6811","nonce":994342538}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6813","nonce":994342538}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6817","nonce":994342538}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6815","nonce":994342538}]},"public_addr":"172.21.15.152:6811/994342538","cluster_addr":"172.21.15.152:6813/994342538","heartbeat_back_addr":"172.21.15.152:6817/994342538","heartbeat_front_addr":"172.21.15.152:6815/994342538","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":101,"up_from":105,"up_thru":110,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6819","nonce":2291823084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6821","nonce":2291823084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6825","nonce":2291823084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6823","nonce":2291823084}]},"public_addr":"172.21.15.152:6819/2291823084","cluster_addr":"172.21.15.152:6821/2291823084","heartbeat_back_addr":"172.21.15.152:6825/2291823084","heartbeat_front_addr":"172.21.15.152:6823/2291823084","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":106,"up_from":110,"up_thru":110,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6827","nonce":3036260714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6829","nonce":3036260714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6833","nonce":3036260714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6831","nonce":3036260714}]},"public_addr":"172.21.15.152:6827/3036260714","cluster_addr":"172.21.15.152:6829/3036260714","heartbeat_back_addr":"172.21.15.152:6833/3036260714","heartbeat_front_addr":"172.21.15.152:6831/3036260714","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":101},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":106},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/1107620726":"2024-04-27T04:58:49.276863+0000","172.21.15.152:0/2072744048":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/1523725454":"2024-04-27T04:54:51.271123+0000","172.21.15.195:6835/3701046555":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.195:6834/3701046555":"2024-04-27T04:59:46.120467+0000","172.21.15.152:6800/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/3065423551":"2024-04-27T05:00:32.300845+0000","172.21.15.152:6801/1036587324":"2024-04-27T04:58:49.276863+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.195:0/244338710":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:6800/982362535":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/4220138504":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/2161488710":"2024-04-27T04:54:51.271123+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:6801/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.195:0/697364496":"2024-04-27T04:59:46.120467+0000","172.21.15.152:6800/1036587324":"2024-04-27T04:58:49.276863+0000","172.21.15.152:0/3569652356":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/1522907277":"2024-04-27T05:00:32.300845+0000","172.21.15.152:6801/982362535":"2024-04-27T05:00:32.300845+0000","172.21.15.195:0/2285242933":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/604484318":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1697576122":"2024-04-27T05:00:32.300845+0000","172.21.15.195:0/4132809147":"2024-04-27T04:59:46.120467+0000","172.21.15.195:0/2956755388":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/2636634813":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/3664778945":"2024-04-27T04:58:49.276863+0000","172.21.15.152:0/3677441985":"2024-04-27T04:58:49.276863+0000","172.21.15.195:0/2882805198":"2024-04-27T04:59:46.120467+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3193673978":"2024-04-27T04:58:49.276863+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:29.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:29.282 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:28 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/517070362' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.292 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/517070362' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.293 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:29.294 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:29.294 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.294 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:29.294 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:06:29.294 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:28 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/517070362' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:06:29.317 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T05:06:29.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:06:29.570 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:31.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:31 smithi152 ceph-mon[102000]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:31.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:31 smithi152 ceph-mon[106538]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:31.473 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:06:31.473 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":111,"fsid":"164326a4-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:49:16.054760+0000","modified":"2024-04-26T05:06:10.273728+0000","last_up_change":"2024-04-26T05:06:09.269506+0000","last_in_change":"2024-04-26T04:54:02.783703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:52:21.033317+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:54:53.698027+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":86,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:56:07.054314+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:56:07.629218+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a2020d0b-1912-45d9-82d2-35664f684411","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":95,"up_thru":110,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6803","nonce":309583967}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6805","nonce":309583967}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6809","nonce":309583967}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":309583967},{"type":"v1","addr":"172.21.15.152:6807","nonce":309583967}]},"public_addr":"172.21.15.152:6803/309583967","cluster_addr":"172.21.15.152:6805/309583967","heartbeat_back_addr":"172.21.15.152:6809/309583967","heartbeat_front_addr":"172.21.15.152:6807/309583967","state":["exists","up"]},{"osd":1,"uuid":"e15a8724-faa2-4271-bb15-5cc3d6886ee7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":100,"up_thru":110,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6811","nonce":994342538}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6813","nonce":994342538}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6817","nonce":994342538}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":994342538},{"type":"v1","addr":"172.21.15.152:6815","nonce":994342538}]},"public_addr":"172.21.15.152:6811/994342538","cluster_addr":"172.21.15.152:6813/994342538","heartbeat_back_addr":"172.21.15.152:6817/994342538","heartbeat_front_addr":"172.21.15.152:6815/994342538","state":["exists","up"]},{"osd":2,"uuid":"3d3029a0-ea36-41c5-b415-20c0ff079f0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":101,"up_from":105,"up_thru":110,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6819","nonce":2291823084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6821","nonce":2291823084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6825","nonce":2291823084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2291823084},{"type":"v1","addr":"172.21.15.152:6823","nonce":2291823084}]},"public_addr":"172.21.15.152:6819/2291823084","cluster_addr":"172.21.15.152:6821/2291823084","heartbeat_back_addr":"172.21.15.152:6825/2291823084","heartbeat_front_addr":"172.21.15.152:6823/2291823084","state":["exists","up"]},{"osd":3,"uuid":"482b6933-60c5-4a4f-a4e9-8365825c20b7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":106,"up_from":110,"up_thru":110,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6827","nonce":3036260714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6829","nonce":3036260714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6832","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6833","nonce":3036260714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":3036260714},{"type":"v1","addr":"172.21.15.152:6831","nonce":3036260714}]},"public_addr":"172.21.15.152:6827/3036260714","cluster_addr":"172.21.15.152:6829/3036260714","heartbeat_back_addr":"172.21.15.152:6833/3036260714","heartbeat_front_addr":"172.21.15.152:6831/3036260714","state":["exists","up"]},{"osd":4,"uuid":"8f134850-9241-4b01-9333-d5a46084c312","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6801","nonce":3775845031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6803","nonce":3775845031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6807","nonce":3775845031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":3775845031},{"type":"v1","addr":"172.21.15.195:6805","nonce":3775845031}]},"public_addr":"172.21.15.195:6801/3775845031","cluster_addr":"172.21.15.195:6803/3775845031","heartbeat_back_addr":"172.21.15.195:6807/3775845031","heartbeat_front_addr":"172.21.15.195:6805/3775845031","state":["exists","up"]},{"osd":5,"uuid":"ab6ff913-82f5-4abf-a461-cb0a7216ef01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6809","nonce":3668288311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6811","nonce":3668288311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6815","nonce":3668288311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3668288311},{"type":"v1","addr":"172.21.15.195:6813","nonce":3668288311}]},"public_addr":"172.21.15.195:6809/3668288311","cluster_addr":"172.21.15.195:6811/3668288311","heartbeat_back_addr":"172.21.15.195:6815/3668288311","heartbeat_front_addr":"172.21.15.195:6813/3668288311","state":["exists","up"]},{"osd":6,"uuid":"c937359c-e6b6-4e0e-afc7-236848f8cd81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6817","nonce":751496603}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6819","nonce":751496603}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6823","nonce":751496603}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":751496603},{"type":"v1","addr":"172.21.15.195:6821","nonce":751496603}]},"public_addr":"172.21.15.195:6817/751496603","cluster_addr":"172.21.15.195:6819/751496603","heartbeat_back_addr":"172.21.15.195:6823/751496603","heartbeat_front_addr":"172.21.15.195:6821/751496603","state":["exists","up"]},{"osd":7,"uuid":"4449492c-2bd2-46c1-96ca-7f929946e4fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6825","nonce":4077645800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6827","nonce":4077645800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6831","nonce":4077645800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":4077645800},{"type":"v1","addr":"172.21.15.195:6829","nonce":4077645800}]},"public_addr":"172.21.15.195:6825/4077645800","cluster_addr":"172.21.15.195:6827/4077645800","heartbeat_back_addr":"172.21.15.195:6831/4077645800","heartbeat_front_addr":"172.21.15.195:6829/4077645800","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:27.472710+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:52.406103+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:17.782448+0000","dead_epoch":101},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:43.197718+0000","dead_epoch":106},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:05.356359+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:29.454607+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:53.484462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:54:18.305363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/1107620726":"2024-04-27T04:58:49.276863+0000","172.21.15.152:0/2072744048":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/2704649277":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/1523725454":"2024-04-27T04:54:51.271123+0000","172.21.15.195:6835/3701046555":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/648205465":"2024-04-27T04:49:41.656061+0000","172.21.15.152:6801/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.195:6834/3701046555":"2024-04-27T04:59:46.120467+0000","172.21.15.152:6800/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/3065423551":"2024-04-27T05:00:32.300845+0000","172.21.15.152:6801/1036587324":"2024-04-27T04:58:49.276863+0000","172.21.15.152:6800/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/3125655303":"2024-04-27T04:50:06.722334+0000","172.21.15.195:0/244338710":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/240502275":"2024-04-27T04:49:41.656061+0000","172.21.15.152:6800/982362535":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/4220138504":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/2161488710":"2024-04-27T04:54:51.271123+0000","172.21.15.152:6800/3284341191":"2024-04-27T04:49:41.656061+0000","172.21.15.152:6801/2475058685":"2024-04-27T04:54:51.271123+0000","172.21.15.195:0/697364496":"2024-04-27T04:59:46.120467+0000","172.21.15.152:6800/1036587324":"2024-04-27T04:58:49.276863+0000","172.21.15.152:0/3569652356":"2024-04-27T05:00:32.300845+0000","172.21.15.152:0/1522907277":"2024-04-27T05:00:32.300845+0000","172.21.15.152:6801/982362535":"2024-04-27T05:00:32.300845+0000","172.21.15.195:0/2285242933":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/604484318":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/1697576122":"2024-04-27T05:00:32.300845+0000","172.21.15.195:0/4132809147":"2024-04-27T04:59:46.120467+0000","172.21.15.195:0/2956755388":"2024-04-27T04:59:46.120467+0000","172.21.15.152:0/2636634813":"2024-04-27T04:54:51.271123+0000","172.21.15.152:0/3664778945":"2024-04-27T04:58:49.276863+0000","172.21.15.152:0/3677441985":"2024-04-27T04:58:49.276863+0000","172.21.15.195:0/2882805198":"2024-04-27T04:59:46.120467+0000","172.21.15.152:6801/1697699354":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/1258912468":"2024-04-27T04:50:06.722334+0000","172.21.15.152:0/4104572086":"2024-04-27T04:49:41.656061+0000","172.21.15.152:0/3193673978":"2024-04-27T04:58:49.276863+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:06:31.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:31 smithi195 ceph-mon[90222]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:32.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T05:06:32.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T05:06:32.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T05:06:32.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T05:06:32.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T05:06:32.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T05:06:32.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T05:06:32.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T05:06:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:32 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/312125136' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:06:32.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:32 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/312125136' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:06:32.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:32 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/312125136' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:06:32.684 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:32.685 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:32.685 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:32.686 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:32.690 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:32.690 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:32.720 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:32.738 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:33.455 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:33 smithi152 ceph-mon[106538]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:33.455 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:33.455 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:33 smithi152 ceph-mon[102000]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:33.455 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:33.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:33 smithi195 ceph-mon[90222]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:33.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:34.331 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:06:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:06:34] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:06:35.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:35 smithi152 ceph-mon[102000]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:35.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:35 smithi152 ceph-mon[106538]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:35.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:35 smithi195 ceph-mon[90222]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:37.302 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:37 smithi152 ceph-mon[106538]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:37.303 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:37 smithi152 ceph-mon[102000]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:37.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:37 smithi195 ceph-mon[90222]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:39.348 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:39 smithi152 ceph-mon[106538]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:39.348 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:39 smithi152 ceph-mon[102000]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:39.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:39 smithi195 ceph-mon[90222]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:41.298 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:41 smithi152 ceph-mon[102000]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:41.299 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:41 smithi152 ceph-mon[106538]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:41.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:41 smithi195 ceph-mon[90222]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:43.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:43 smithi195 ceph-mon[90222]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:43.542 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:43 smithi152 ceph-mon[106538]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:43.542 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:43 smithi152 ceph-mon[102000]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:44.460 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:06:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:06:44] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:06:45.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:45 smithi152 ceph-mon[102000]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:45.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:45 smithi152 ceph-mon[106538]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:45.492 INFO:teuthology.orchestra.run.smithi152.stdout:472446402568 2024-04-26T05:06:45.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T05:06:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:45 smithi195 ceph-mon[90222]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:45.884 INFO:teuthology.orchestra.run.smithi152.stdout:236223201431 2024-04-26T05:06:45.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T05:06:46.577 INFO:teuthology.orchestra.run.smithi152.stdout:133143986342 2024-04-26T05:06:46.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T05:06:47.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:47 smithi195 ceph-mon[90222]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:47.597 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:47 smithi152 ceph-mon[106538]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:47.597 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:47 smithi152 ceph-mon[102000]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:48.163 INFO:teuthology.orchestra.run.smithi152.stdout:167503724706 2024-04-26T05:06:48.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T05:06:48.389 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:48.390 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:48.886 INFO:teuthology.orchestra.run.smithi152.stdout:450971566096 2024-04-26T05:06:48.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T05:06:49.021 INFO:teuthology.orchestra.run.smithi152.stdout:429496729623 2024-04-26T05:06:49.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T05:06:49.350 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:49 smithi152 ceph-mon[106538]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:49.350 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:49 smithi152 ceph-mon[102000]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:49.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:49 smithi195 ceph-mon[90222]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:49.665 INFO:teuthology.orchestra.run.smithi152.stdout:408021893157 2024-04-26T05:06:49.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T05:06:50.112 INFO:teuthology.orchestra.run.smithi152.stdout:201863463071 2024-04-26T05:06:50.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T05:06:50.200 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:50.200 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:50.203 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:50.203 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:50.269 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:50.270 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:51.482 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:51 smithi152 ceph-mon[102000]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:51.482 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:51 smithi152 ceph-mon[106538]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:51.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:51 smithi195 ceph-mon[90222]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:53.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:53 smithi152 ceph-mon[102000]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:53.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:53 smithi152 ceph-mon[106538]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:53.428 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:53.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:53 smithi195 ceph-mon[90222]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:54.371 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:06:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:06:54] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2024-04-26T05:06:55.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:55 smithi195 ceph-mon[90222]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:55.600 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:55 smithi152 ceph-mon[102000]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:55.600 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:55 smithi152 ceph-mon[106538]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.335 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:57.498 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:57 smithi152 ceph-mon[106538]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.499 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:57 smithi152 ceph-mon[102000]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:57 smithi195 ceph-mon[90222]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.553 INFO:teuthology.orchestra.run.smithi152.stdout:236223201434 2024-04-26T05:06:57.840 INFO:teuthology.orchestra.run.smithi152.stdout:167503724708 2024-04-26T05:06:58.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:58 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/929676641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:06:58.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:58 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2276031988' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:06:58.606 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:58 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/929676641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:06:58.606 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:58 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2276031988' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:06:58.606 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:58 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/929676641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:06:58.606 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:58 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2276031988' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:06:59.479 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:06:59 smithi152 ceph-mon[102000]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:59.480 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:06:59 smithi152 ceph-mon[106538]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:59.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:06:59 smithi195 ceph-mon[90222]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:00.986 INFO:teuthology.orchestra.run.smithi152.stdout:408021893160 2024-04-26T05:07:01.294 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724706 got 167503724708 for osd.5 2024-04-26T05:07:01.294 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:01.504 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:01 smithi152 ceph-mon[102000]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.504 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:01 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/4076403994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:07:01.504 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:01 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/679094764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:07:01.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:01 smithi152 ceph-mon[106538]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:01 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/4076403994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:07:01.505 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:01 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/679094764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:07:01.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:01 smithi195 ceph-mon[90222]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:01 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/4076403994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:07:01.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:01 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/679094764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:07:01.722 INFO:teuthology.orchestra.run.smithi152.stdout:450971566099 2024-04-26T05:07:01.954 INFO:teuthology.orchestra.run.smithi152.stdout:133143986346 2024-04-26T05:07:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:02 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/974971882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:07:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:02 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2421194216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:07:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:02 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/974971882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:07:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:02 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2421194216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:07:02.420 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201431 got 236223201434 for osd.7 2024-04-26T05:07:02.420 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:02.503 INFO:teuthology.orchestra.run.smithi152.stdout:429496729626 2024-04-26T05:07:02.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:02 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/974971882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:07:02.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:02 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2421194216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:07:03.236 INFO:tasks.cephadm.ceph_manager.ceph:need seq 429496729623 got 429496729626 for osd.1 2024-04-26T05:07:03.236 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:03 smithi195 ceph-mon[90222]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:03.552 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:03 smithi152 ceph-mon[102000]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:03.552 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:03.553 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:03 smithi152 ceph-mon[106538]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:03.553 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:03.876 INFO:teuthology.orchestra.run.smithi152.stdout:472446402572 2024-04-26T05:07:04.079 INFO:tasks.cephadm.ceph_manager.ceph:need seq 408021893157 got 408021893160 for osd.0 2024-04-26T05:07:04.079 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:04 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1442720203' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:07:04.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:04 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1442720203' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:07:04.613 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:07:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:07:04] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.43.0" 2024-04-26T05:07:04.614 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:04 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1442720203' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:07:04.632 INFO:tasks.cephadm.ceph_manager.ceph:need seq 472446402568 got 472446402572 for osd.3 2024-04-26T05:07:04.632 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:05.137 INFO:tasks.cephadm.ceph_manager.ceph:need seq 450971566096 got 450971566099 for osd.2 2024-04-26T05:07:05.137 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:05.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:05 smithi152 ceph-mon[102000]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:05 smithi152 ceph-mon[106538]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:05 smithi195 ceph-mon[90222]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:06.644 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986342 got 133143986346 for osd.4 2024-04-26T05:07:06.644 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:07.511 INFO:teuthology.orchestra.run.smithi152.stdout:201863463074 2024-04-26T05:07:07.511 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:07 smithi152 ceph-mon[102000]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:07.512 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:07 smithi152 ceph-mon[106538]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:07.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:07 smithi195 ceph-mon[90222]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:08.198 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463071 got 201863463074 for osd.6 2024-04-26T05:07:08.198 DEBUG:teuthology.parallel:result is None 2024-04-26T05:07:08.199 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T05:07:08.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:07:08.460 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:08.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:08 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/291334244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:07:08.581 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:08 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/291334244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:07:08.582 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:08 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/291334244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:07:09.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:09 smithi195 ceph-mon[90222]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:09.606 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:09 smithi152 ceph-mon[102000]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:09.606 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:09 smithi152 ceph-mon[106538]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:10.329 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-26T05:07:10.994 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":244,"stamp":"2024-04-26T05:07:10.146204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6494,"num_read_kb":12908,"num_write":1146,"num_write_kb":15039,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1335604,"kb_used_data":23612,"kb_used_omap":36,"kb_used_meta":1311835,"kb_avail":748625612,"statfs":{"total":767960285184,"available":766592626688,"internally_reserved":0,"allocated":24178688,"data_stored":19440506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37579,"internal_metadata":1343319349},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002696"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275046+0000","last_change":"2024-04-26T05:05:01.307763+0000","last_active":"2024-04-26T05:06:09.275046+0000","last_peered":"2024-04-26T05:06:09.275046+0000","last_clean":"2024-04-26T05:06:09.275046+0000","last_became_active":"2024-04-26T05:05:01.307169+0000","last_became_peered":"2024-04-26T05:05:01.307169+0000","last_unstale":"2024-04-26T05:06:09.275046+0000","last_undegraded":"2024-04-26T05:06:09.275046+0000","last_fullsized":"2024-04-26T05:06:09.275046+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:18:58.255124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":66,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283517+0000","last_change":"2024-04-26T05:05:35.493186+0000","last_active":"2024-04-26T05:06:10.283517+0000","last_peered":"2024-04-26T05:06:10.283517+0000","last_clean":"2024-04-26T05:06:10.283517+0000","last_became_active":"2024-04-26T05:05:35.492807+0000","last_became_peered":"2024-04-26T05:05:35.492807+0000","last_unstale":"2024-04-26T05:06:10.283517+0000","last_undegraded":"2024-04-26T05:06:10.283517+0000","last_fullsized":"2024-04-26T05:06:10.283517+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:26:40.440227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296310+0000","last_change":"2024-04-26T05:06:10.296310+0000","last_active":"2024-04-26T05:06:10.296310+0000","last_peered":"2024-04-26T05:06:10.296310+0000","last_clean":"2024-04-26T05:06:10.296310+0000","last_became_active":"2024-04-26T05:06:10.295934+0000","last_became_peered":"2024-04-26T05:06:10.295934+0000","last_unstale":"2024-04-26T05:06:10.296310+0000","last_undegraded":"2024-04-26T05:06:10.296310+0000","last_fullsized":"2024-04-26T05:06:10.296310+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:17:11.064327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":78,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294678+0000","last_change":"2024-04-26T05:06:10.294678+0000","last_active":"2024-04-26T05:06:10.294678+0000","last_peered":"2024-04-26T05:06:10.294678+0000","last_clean":"2024-04-26T05:06:10.294678+0000","last_became_active":"2024-04-26T05:06:10.294276+0000","last_became_peered":"2024-04-26T05:06:10.294276+0000","last_unstale":"2024-04-26T05:06:10.294678+0000","last_undegraded":"2024-04-26T05:06:10.294678+0000","last_fullsized":"2024-04-26T05:06:10.294678+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:34:06.294703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288581+0000","last_change":"2024-04-26T05:06:10.288581+0000","last_active":"2024-04-26T05:06:10.288581+0000","last_peered":"2024-04-26T05:06:10.288581+0000","last_clean":"2024-04-26T05:06:10.288581+0000","last_became_active":"2024-04-26T05:06:10.288223+0000","last_became_peered":"2024-04-26T05:06:10.288223+0000","last_unstale":"2024-04-26T05:06:10.288581+0000","last_undegraded":"2024-04-26T05:06:10.288581+0000","last_fullsized":"2024-04-26T05:06:10.288581+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:51:27.080918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280928+0000","last_change":"2024-04-26T04:56:08.400207+0000","last_active":"2024-04-26T05:06:09.280928+0000","last_peered":"2024-04-26T05:06:09.280928+0000","last_clean":"2024-04-26T05:06:09.280928+0000","last_became_active":"2024-04-26T04:56:08.399958+0000","last_became_peered":"2024-04-26T04:56:08.399958+0000","last_unstale":"2024-04-26T05:06:09.280928+0000","last_undegraded":"2024-04-26T05:06:09.280928+0000","last_fullsized":"2024-04-26T05:06:09.280928+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:58:07.756276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288545+0000","last_change":"2024-04-26T05:06:10.288545+0000","last_active":"2024-04-26T05:06:10.288545+0000","last_peered":"2024-04-26T05:06:10.288545+0000","last_clean":"2024-04-26T05:06:10.288545+0000","last_became_active":"2024-04-26T05:06:10.287834+0000","last_became_peered":"2024-04-26T05:06:10.287834+0000","last_unstale":"2024-04-26T05:06:10.288545+0000","last_undegraded":"2024-04-26T05:06:10.288545+0000","last_fullsized":"2024-04-26T05:06:10.288545+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:39:29.678078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278382+0000","last_change":"2024-04-26T05:05:01.306665+0000","last_active":"2024-04-26T05:06:09.278382+0000","last_peered":"2024-04-26T05:06:09.278382+0000","last_clean":"2024-04-26T05:06:09.278382+0000","last_became_active":"2024-04-26T05:05:01.306322+0000","last_became_peered":"2024-04-26T05:05:01.306322+0000","last_unstale":"2024-04-26T05:06:09.278382+0000","last_undegraded":"2024-04-26T05:06:09.278382+0000","last_fullsized":"2024-04-26T05:06:09.278382+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:34:24.045728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292007+0000","last_change":"2024-04-26T05:06:10.292007+0000","last_active":"2024-04-26T05:06:10.292007+0000","last_peered":"2024-04-26T05:06:10.292007+0000","last_clean":"2024-04-26T05:06:10.292007+0000","last_became_active":"2024-04-26T05:06:10.291651+0000","last_became_peered":"2024-04-26T05:06:10.291651+0000","last_unstale":"2024-04-26T05:06:10.292007+0000","last_undegraded":"2024-04-26T05:06:10.292007+0000","last_fullsized":"2024-04-26T05:06:10.292007+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:38:11.322567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":57,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.284072+0000","last_change":"2024-04-26T05:03:50.868029+0000","last_active":"2024-04-26T05:06:10.284072+0000","last_peered":"2024-04-26T05:06:10.284072+0000","last_clean":"2024-04-26T05:06:10.284072+0000","last_became_active":"2024-04-26T05:03:50.867549+0000","last_became_peered":"2024-04-26T05:03:50.867549+0000","last_unstale":"2024-04-26T05:06:10.284072+0000","last_undegraded":"2024-04-26T05:06:10.284072+0000","last_fullsized":"2024-04-26T05:06:10.284072+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:21:59.272714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.277125+0000","last_change":"2024-04-26T04:56:08.417990+0000","last_active":"2024-04-26T05:06:09.277125+0000","last_peered":"2024-04-26T05:06:09.277125+0000","last_clean":"2024-04-26T05:06:09.277125+0000","last_became_active":"2024-04-26T04:56:08.417703+0000","last_became_peered":"2024-04-26T04:56:08.417703+0000","last_unstale":"2024-04-26T05:06:09.277125+0000","last_undegraded":"2024-04-26T05:06:09.277125+0000","last_fullsized":"2024-04-26T05:06:09.277125+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:49:52.041048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":66,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283769+0000","last_change":"2024-04-26T05:05:01.307393+0000","last_active":"2024-04-26T05:06:10.283769+0000","last_peered":"2024-04-26T05:06:10.283769+0000","last_clean":"2024-04-26T05:06:10.283769+0000","last_became_active":"2024-04-26T05:05:01.307007+0000","last_became_peered":"2024-04-26T05:05:01.307007+0000","last_unstale":"2024-04-26T05:06:10.283769+0000","last_undegraded":"2024-04-26T05:06:10.283769+0000","last_fullsized":"2024-04-26T05:06:10.283769+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:51:50.722261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"87'97","reported_seq":441,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.279971+0000","last_change":"2024-04-26T05:05:35.492969+0000","last_active":"2024-04-26T05:06:09.279971+0000","last_peered":"2024-04-26T05:06:09.279971+0000","last_clean":"2024-04-26T05:06:09.279971+0000","last_became_active":"2024-04-26T05:05:35.492441+0000","last_became_peered":"2024-04-26T05:05:35.492441+0000","last_unstale":"2024-04-26T05:06:09.279971+0000","last_undegraded":"2024-04-26T05:06:09.279971+0000","last_fullsized":"2024-04-26T05:06:09.279971+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:45:40.923654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087697999999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":696,"num_read_kb":866,"num_write":90,"num_write_kb":5299,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296523+0000","last_change":"2024-04-26T05:06:10.296523+0000","last_active":"2024-04-26T05:06:10.296523+0000","last_peered":"2024-04-26T05:06:10.296523+0000","last_clean":"2024-04-26T05:06:10.296523+0000","last_became_active":"2024-04-26T05:06:10.296132+0000","last_became_peered":"2024-04-26T05:06:10.296132+0000","last_unstale":"2024-04-26T05:06:10.296523+0000","last_undegraded":"2024-04-26T05:06:10.296523+0000","last_fullsized":"2024-04-26T05:06:10.296523+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:41.356448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"87'105","reported_seq":320,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280655+0000","last_change":"2024-04-26T05:05:01.303861+0000","last_active":"2024-04-26T05:06:10.280655+0000","last_peered":"2024-04-26T05:06:10.280655+0000","last_clean":"2024-04-26T05:06:10.280655+0000","last_became_active":"2024-04-26T05:05:01.303177+0000","last_became_peered":"2024-04-26T05:05:01.303177+0000","last_unstale":"2024-04-26T05:06:10.280655+0000","last_undegraded":"2024-04-26T05:06:10.280655+0000","last_fullsized":"2024-04-26T05:06:10.280655+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:58:38.326483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022107600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":369,"num_read_kb":388,"num_write":84,"num_write_kb":1127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296560+0000","last_change":"2024-04-26T05:06:10.296560+0000","last_active":"2024-04-26T05:06:10.296560+0000","last_peered":"2024-04-26T05:06:10.296560+0000","last_clean":"2024-04-26T05:06:10.296560+0000","last_became_active":"2024-04-26T05:06:10.296000+0000","last_became_peered":"2024-04-26T05:06:10.296000+0000","last_unstale":"2024-04-26T05:06:10.296560+0000","last_undegraded":"2024-04-26T05:06:10.296560+0000","last_fullsized":"2024-04-26T05:06:10.296560+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:17:30.714197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283829+0000","last_change":"2024-04-26T05:03:50.865358+0000","last_active":"2024-04-26T05:06:10.283829+0000","last_peered":"2024-04-26T05:06:10.283829+0000","last_clean":"2024-04-26T05:06:10.283829+0000","last_became_active":"2024-04-26T05:03:50.864950+0000","last_became_peered":"2024-04-26T05:03:50.864950+0000","last_unstale":"2024-04-26T05:06:10.283829+0000","last_undegraded":"2024-04-26T05:06:10.283829+0000","last_fullsized":"2024-04-26T05:06:10.283829+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:57:40.431639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280681+0000","last_change":"2024-04-26T05:05:35.494290+0000","last_active":"2024-04-26T05:06:10.280681+0000","last_peered":"2024-04-26T05:06:10.280681+0000","last_clean":"2024-04-26T05:06:10.280681+0000","last_became_active":"2024-04-26T05:05:35.493789+0000","last_became_peered":"2024-04-26T05:05:35.493789+0000","last_unstale":"2024-04-26T05:06:10.280681+0000","last_undegraded":"2024-04-26T05:06:10.280681+0000","last_fullsized":"2024-04-26T05:06:10.280681+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:46:00.079348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"87'36","reported_seq":247,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280434+0000","last_change":"2024-04-26T05:03:50.870038+0000","last_active":"2024-04-26T05:06:09.280434+0000","last_peered":"2024-04-26T05:06:09.280434+0000","last_clean":"2024-04-26T05:06:09.280434+0000","last_became_active":"2024-04-26T05:03:50.869602+0000","last_became_peered":"2024-04-26T05:03:50.869602+0000","last_unstale":"2024-04-26T05:06:09.280434+0000","last_undegraded":"2024-04-26T05:06:09.280434+0000","last_fullsized":"2024-04-26T05:06:09.280434+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":36,"log_dups_size":0,"ondisk_log_size":36,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:44:48.772118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031756800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":214,"num_read_kb":1209,"num_write":25,"num_write_kb":1044,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283916+0000","last_change":"2024-04-26T05:05:35.493573+0000","last_active":"2024-04-26T05:06:10.283916+0000","last_peered":"2024-04-26T05:06:10.283916+0000","last_clean":"2024-04-26T05:06:10.283916+0000","last_became_active":"2024-04-26T05:05:35.493241+0000","last_became_peered":"2024-04-26T05:05:35.493241+0000","last_unstale":"2024-04-26T05:06:10.283916+0000","last_undegraded":"2024-04-26T05:06:10.283916+0000","last_fullsized":"2024-04-26T05:06:10.283916+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:33.340797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292519+0000","last_change":"2024-04-26T05:06:10.292519+0000","last_active":"2024-04-26T05:06:10.292519+0000","last_peered":"2024-04-26T05:06:10.292519+0000","last_clean":"2024-04-26T05:06:10.292519+0000","last_became_active":"2024-04-26T05:06:10.292205+0000","last_became_peered":"2024-04-26T05:06:10.292205+0000","last_unstale":"2024-04-26T05:06:10.292519+0000","last_undegraded":"2024-04-26T05:06:10.292519+0000","last_fullsized":"2024-04-26T05:06:10.292519+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:48:57.035079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"87'88","reported_seq":436,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282363+0000","last_change":"2024-04-26T05:05:01.305251+0000","last_active":"2024-04-26T05:06:09.282363+0000","last_peered":"2024-04-26T05:06:09.282363+0000","last_clean":"2024-04-26T05:06:09.282363+0000","last_became_active":"2024-04-26T05:05:01.304749+0000","last_became_peered":"2024-04-26T05:05:01.304749+0000","last_unstale":"2024-04-26T05:06:09.282363+0000","last_undegraded":"2024-04-26T05:06:09.282363+0000","last_fullsized":"2024-04-26T05:06:09.282363+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:14:35.322089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038853800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":510,"num_write":72,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282133+0000","last_change":"2024-04-26T04:56:08.735555+0000","last_active":"2024-04-26T05:06:09.282133+0000","last_peered":"2024-04-26T05:06:09.282133+0000","last_clean":"2024-04-26T05:06:09.282133+0000","last_became_active":"2024-04-26T04:56:08.735298+0000","last_became_peered":"2024-04-26T04:56:08.735298+0000","last_unstale":"2024-04-26T05:06:09.282133+0000","last_undegraded":"2024-04-26T05:06:09.282133+0000","last_fullsized":"2024-04-26T05:06:09.282133+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:08.329803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294348+0000","last_change":"2024-04-26T05:06:10.294348+0000","last_active":"2024-04-26T05:06:10.294348+0000","last_peered":"2024-04-26T05:06:10.294348+0000","last_clean":"2024-04-26T05:06:10.294348+0000","last_became_active":"2024-04-26T05:06:10.294038+0000","last_became_peered":"2024-04-26T05:06:10.294038+0000","last_unstale":"2024-04-26T05:06:10.294348+0000","last_undegraded":"2024-04-26T05:06:10.294348+0000","last_fullsized":"2024-04-26T05:06:10.294348+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:12.400548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":450,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282266+0000","last_change":"2024-04-26T05:03:50.869031+0000","last_active":"2024-04-26T05:06:09.282266+0000","last_peered":"2024-04-26T05:06:09.282266+0000","last_clean":"2024-04-26T05:06:09.282266+0000","last_became_active":"2024-04-26T05:03:50.868240+0000","last_became_peered":"2024-04-26T05:03:50.868240+0000","last_unstale":"2024-04-26T05:06:09.282266+0000","last_undegraded":"2024-04-26T05:06:09.282266+0000","last_fullsized":"2024-04-26T05:06:09.282266+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_clean_scrub_stamp":"2024-04-26T04:52:21.376988+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:51:08.168877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"87'216","reported_seq":760,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278214+0000","last_change":"2024-04-26T05:05:35.493996+0000","last_active":"2024-04-26T05:06:09.278214+0000","last_peered":"2024-04-26T05:06:09.278214+0000","last_clean":"2024-04-26T05:06:09.278214+0000","last_became_active":"2024-04-26T05:05:35.493332+0000","last_became_peered":"2024-04-26T05:05:35.493332+0000","last_unstale":"2024-04-26T05:06:09.278214+0000","last_undegraded":"2024-04-26T05:06:09.278214+0000","last_fullsized":"2024-04-26T05:06:09.278214+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":216,"log_dups_size":0,"ondisk_log_size":216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:03:37.684836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000304054,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1035,"num_read_kb":923,"num_write":190,"num_write_kb":120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280224+0000","last_change":"2024-04-26T05:05:01.306363+0000","last_active":"2024-04-26T05:06:10.280224+0000","last_peered":"2024-04-26T05:06:10.280224+0000","last_clean":"2024-04-26T05:06:10.280224+0000","last_became_active":"2024-04-26T05:05:01.304618+0000","last_became_peered":"2024-04-26T05:05:01.304618+0000","last_unstale":"2024-04-26T05:06:10.280224+0000","last_undegraded":"2024-04-26T05:06:10.280224+0000","last_fullsized":"2024-04-26T05:06:10.280224+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:17.531286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280076+0000","last_change":"2024-04-26T05:05:35.492906+0000","last_active":"2024-04-26T05:06:10.280076+0000","last_peered":"2024-04-26T05:06:10.280076+0000","last_clean":"2024-04-26T05:06:10.280076+0000","last_became_active":"2024-04-26T05:05:35.492594+0000","last_became_peered":"2024-04-26T05:05:35.492594+0000","last_unstale":"2024-04-26T05:06:10.280076+0000","last_undegraded":"2024-04-26T05:06:10.280076+0000","last_fullsized":"2024-04-26T05:06:10.280076+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:14:12.268165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"87'190","reported_seq":825,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.293037+0000","last_change":"2024-04-26T05:06:10.293037+0000","last_active":"2024-04-26T05:06:10.293037+0000","last_peered":"2024-04-26T05:06:10.293037+0000","last_clean":"2024-04-26T05:06:10.293037+0000","last_became_active":"2024-04-26T05:06:10.292403+0000","last_became_peered":"2024-04-26T05:06:10.292403+0000","last_unstale":"2024-04-26T05:06:10.293037+0000","last_undegraded":"2024-04-26T05:06:10.293037+0000","last_fullsized":"2024-04-26T05:06:10.293037+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:46.240751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000392913,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1328,"num_read_kb":3434,"num_write":163,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289550+0000","last_change":"2024-04-26T05:06:10.288557+0000","last_active":"2024-04-26T05:06:10.289550+0000","last_peered":"2024-04-26T05:06:10.289550+0000","last_clean":"2024-04-26T05:06:10.289550+0000","last_became_active":"2024-04-26T05:06:10.287948+0000","last_became_peered":"2024-04-26T05:06:10.287948+0000","last_unstale":"2024-04-26T05:06:10.289550+0000","last_undegraded":"2024-04-26T05:06:10.289550+0000","last_fullsized":"2024-04-26T05:06:10.289550+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:09:22.728442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275314+0000","last_change":"2024-04-26T05:03:50.866118+0000","last_active":"2024-04-26T05:06:09.275314+0000","last_peered":"2024-04-26T05:06:09.275314+0000","last_clean":"2024-04-26T05:06:09.275314+0000","last_became_active":"2024-04-26T05:03:50.865744+0000","last_became_peered":"2024-04-26T05:03:50.865744+0000","last_unstale":"2024-04-26T05:06:09.275314+0000","last_undegraded":"2024-04-26T05:06:09.275314+0000","last_fullsized":"2024-04-26T05:06:09.275314+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:35:07.013499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"87'77","reported_seq":557,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278133+0000","last_change":"2024-04-26T05:05:01.306063+0000","last_active":"2024-04-26T05:06:09.278133+0000","last_peered":"2024-04-26T05:06:09.278133+0000","last_clean":"2024-04-26T05:06:09.278133+0000","last_became_active":"2024-04-26T05:05:01.305595+0000","last_became_peered":"2024-04-26T05:05:01.305595+0000","last_unstale":"2024-04-26T05:06:09.278133+0000","last_undegraded":"2024-04-26T05:06:09.278133+0000","last_fullsized":"2024-04-26T05:06:09.278133+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":77,"log_dups_size":0,"ondisk_log_size":77,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:24:24.544418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000660861,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":598,"num_read_kb":2063,"num_write":68,"num_write_kb":1301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283265+0000","last_change":"2024-04-26T05:05:35.497639+0000","last_active":"2024-04-26T05:06:10.283265+0000","last_peered":"2024-04-26T05:06:10.283265+0000","last_clean":"2024-04-26T05:06:10.283265+0000","last_became_active":"2024-04-26T05:05:35.496820+0000","last_became_peered":"2024-04-26T05:05:35.496820+0000","last_unstale":"2024-04-26T05:06:10.283265+0000","last_undegraded":"2024-04-26T05:06:10.283265+0000","last_fullsized":"2024-04-26T05:06:10.283265+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:12:15.241870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289553+0000","last_change":"2024-04-26T05:06:10.289553+0000","last_active":"2024-04-26T05:06:10.289553+0000","last_peered":"2024-04-26T05:06:10.289553+0000","last_clean":"2024-04-26T05:06:10.289553+0000","last_became_active":"2024-04-26T05:06:10.289218+0000","last_became_peered":"2024-04-26T05:06:10.289218+0000","last_unstale":"2024-04-26T05:06:10.289553+0000","last_undegraded":"2024-04-26T05:06:10.289553+0000","last_fullsized":"2024-04-26T05:06:10.289553+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:49:07.184501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"87'167","reported_seq":764,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280749+0000","last_change":"2024-04-26T05:05:01.307161+0000","last_active":"2024-04-26T05:06:10.280749+0000","last_peered":"2024-04-26T05:06:10.280749+0000","last_clean":"2024-04-26T05:06:10.280749+0000","last_became_active":"2024-04-26T05:05:01.306323+0000","last_became_peered":"2024-04-26T05:05:01.306323+0000","last_unstale":"2024-04-26T05:06:10.280749+0000","last_undegraded":"2024-04-26T05:06:10.280749+0000","last_fullsized":"2024-04-26T05:06:10.280749+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:17:40.461307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050161400000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1161,"num_read_kb":2591,"num_write":154,"num_write_kb":1373,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289869+0000","last_change":"2024-04-26T05:06:10.289869+0000","last_active":"2024-04-26T05:06:10.289869+0000","last_peered":"2024-04-26T05:06:10.289869+0000","last_clean":"2024-04-26T05:06:10.289869+0000","last_became_active":"2024-04-26T05:06:10.289551+0000","last_became_peered":"2024-04-26T05:06:10.289551+0000","last_unstale":"2024-04-26T05:06:10.289869+0000","last_undegraded":"2024-04-26T05:06:10.289869+0000","last_fullsized":"2024-04-26T05:06:10.289869+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:52:04.722533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292244+0000","last_change":"2024-04-26T05:06:10.292244+0000","last_active":"2024-04-26T05:06:10.292244+0000","last_peered":"2024-04-26T05:06:10.292244+0000","last_clean":"2024-04-26T05:06:10.292244+0000","last_became_active":"2024-04-26T05:06:10.291108+0000","last_became_peered":"2024-04-26T05:06:10.291108+0000","last_unstale":"2024-04-26T05:06:10.292244+0000","last_undegraded":"2024-04-26T05:06:10.292244+0000","last_fullsized":"2024-04-26T05:06:10.292244+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:47:16.984760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.276134+0000","last_change":"2024-04-26T05:05:01.309433+0000","last_active":"2024-04-26T05:06:09.276134+0000","last_peered":"2024-04-26T05:06:09.276134+0000","last_clean":"2024-04-26T05:06:09.276134+0000","last_became_active":"2024-04-26T05:05:01.308770+0000","last_became_peered":"2024-04-26T05:05:01.308770+0000","last_unstale":"2024-04-26T05:06:09.276134+0000","last_undegraded":"2024-04-26T05:06:09.276134+0000","last_fullsized":"2024-04-26T05:06:09.276134+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:34:58.585785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.277209+0000","last_change":"2024-04-26T05:05:35.497619+0000","last_active":"2024-04-26T05:06:09.277209+0000","last_peered":"2024-04-26T05:06:09.277209+0000","last_clean":"2024-04-26T05:06:09.277209+0000","last_became_active":"2024-04-26T05:05:35.497260+0000","last_became_peered":"2024-04-26T05:05:35.497260+0000","last_unstale":"2024-04-26T05:06:09.277209+0000","last_undegraded":"2024-04-26T05:06:09.277209+0000","last_fullsized":"2024-04-26T05:06:09.277209+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:29:28.499889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.276687+0000","last_change":"2024-04-26T05:05:35.495597+0000","last_active":"2024-04-26T05:06:09.276687+0000","last_peered":"2024-04-26T05:06:09.276687+0000","last_clean":"2024-04-26T05:06:09.276687+0000","last_became_active":"2024-04-26T05:05:35.495274+0000","last_became_peered":"2024-04-26T05:05:35.495274+0000","last_unstale":"2024-04-26T05:06:09.276687+0000","last_undegraded":"2024-04-26T05:06:09.276687+0000","last_fullsized":"2024-04-26T05:06:09.276687+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:16.648302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280880+0000","last_change":"2024-04-26T05:05:01.301531+0000","last_active":"2024-04-26T05:06:09.280880+0000","last_peered":"2024-04-26T05:06:09.280880+0000","last_clean":"2024-04-26T05:06:09.280880+0000","last_became_active":"2024-04-26T05:05:01.301149+0000","last_became_peered":"2024-04-26T05:05:01.301149+0000","last_unstale":"2024-04-26T05:06:09.280880+0000","last_undegraded":"2024-04-26T05:06:09.280880+0000","last_fullsized":"2024-04-26T05:06:09.280880+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:49.906225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.291847+0000","last_change":"2024-04-26T05:06:10.291847+0000","last_active":"2024-04-26T05:06:10.291847+0000","last_peered":"2024-04-26T05:06:10.291847+0000","last_clean":"2024-04-26T05:06:10.291847+0000","last_became_active":"2024-04-26T05:06:10.291495+0000","last_became_peered":"2024-04-26T05:06:10.291495+0000","last_unstale":"2024-04-26T05:06:10.291847+0000","last_undegraded":"2024-04-26T05:06:10.291847+0000","last_fullsized":"2024-04-26T05:06:10.291847+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:44.938298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296035+0000","last_change":"2024-04-26T05:06:10.296035+0000","last_active":"2024-04-26T05:06:10.296035+0000","last_peered":"2024-04-26T05:06:10.296035+0000","last_clean":"2024-04-26T05:06:10.296035+0000","last_became_active":"2024-04-26T05:06:10.294898+0000","last_became_peered":"2024-04-26T05:06:10.294898+0000","last_unstale":"2024-04-26T05:06:10.296035+0000","last_undegraded":"2024-04-26T05:06:10.296035+0000","last_fullsized":"2024-04-26T05:06:10.296035+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:18:36.704546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283942+0000","last_change":"2024-04-26T05:05:01.305384+0000","last_active":"2024-04-26T05:06:10.283942+0000","last_peered":"2024-04-26T05:06:10.283942+0000","last_clean":"2024-04-26T05:06:10.283942+0000","last_became_active":"2024-04-26T05:05:01.305046+0000","last_became_peered":"2024-04-26T05:05:01.305046+0000","last_unstale":"2024-04-26T05:06:10.283942+0000","last_undegraded":"2024-04-26T05:06:10.283942+0000","last_fullsized":"2024-04-26T05:06:10.283942+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:26.099419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.291825+0000","last_change":"2024-04-26T05:06:10.291825+0000","last_active":"2024-04-26T05:06:10.291825+0000","last_peered":"2024-04-26T05:06:10.291825+0000","last_clean":"2024-04-26T05:06:10.291825+0000","last_became_active":"2024-04-26T05:06:10.291537+0000","last_became_peered":"2024-04-26T05:06:10.291537+0000","last_unstale":"2024-04-26T05:06:10.291825+0000","last_undegraded":"2024-04-26T05:06:10.291825+0000","last_fullsized":"2024-04-26T05:06:10.291825+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:56:13.081732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280736+0000","last_change":"2024-04-26T05:05:01.299480+0000","last_active":"2024-04-26T05:06:09.280736+0000","last_peered":"2024-04-26T05:06:09.280736+0000","last_clean":"2024-04-26T05:06:09.280736+0000","last_became_active":"2024-04-26T05:05:01.299142+0000","last_became_peered":"2024-04-26T05:05:01.299142+0000","last_unstale":"2024-04-26T05:06:09.280736+0000","last_undegraded":"2024-04-26T05:06:09.280736+0000","last_fullsized":"2024-04-26T05:06:09.280736+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:37:49.299492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280531+0000","last_change":"2024-04-26T04:56:08.400274+0000","last_active":"2024-04-26T05:06:09.280531+0000","last_peered":"2024-04-26T05:06:09.280531+0000","last_clean":"2024-04-26T05:06:09.280531+0000","last_became_active":"2024-04-26T04:56:08.400012+0000","last_became_peered":"2024-04-26T04:56:08.400012+0000","last_unstale":"2024-04-26T05:06:09.280531+0000","last_undegraded":"2024-04-26T05:06:09.280531+0000","last_fullsized":"2024-04-26T05:06:09.280531+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:06:01.413415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288328+0000","last_change":"2024-04-26T05:06:10.288328+0000","last_active":"2024-04-26T05:06:10.288328+0000","last_peered":"2024-04-26T05:06:10.288328+0000","last_clean":"2024-04-26T05:06:10.288328+0000","last_became_active":"2024-04-26T05:06:10.287876+0000","last_became_peered":"2024-04-26T05:06:10.287876+0000","last_unstale":"2024-04-26T05:06:10.288328+0000","last_undegraded":"2024-04-26T05:06:10.288328+0000","last_fullsized":"2024-04-26T05:06:10.288328+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:46:13.556328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280231+0000","last_change":"2024-04-26T05:05:01.305251+0000","last_active":"2024-04-26T05:06:09.280231+0000","last_peered":"2024-04-26T05:06:09.280231+0000","last_clean":"2024-04-26T05:06:09.280231+0000","last_became_active":"2024-04-26T05:05:01.304784+0000","last_became_peered":"2024-04-26T05:05:01.304784+0000","last_unstale":"2024-04-26T05:06:09.280231+0000","last_undegraded":"2024-04-26T05:06:09.280231+0000","last_fullsized":"2024-04-26T05:06:09.280231+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:58:25.361117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278064+0000","last_change":"2024-04-26T04:56:08.735825+0000","last_active":"2024-04-26T05:06:09.278064+0000","last_peered":"2024-04-26T05:06:09.278064+0000","last_clean":"2024-04-26T05:06:09.278064+0000","last_became_active":"2024-04-26T04:56:08.735226+0000","last_became_peered":"2024-04-26T04:56:08.735226+0000","last_unstale":"2024-04-26T05:06:09.278064+0000","last_undegraded":"2024-04-26T05:06:09.278064+0000","last_fullsized":"2024-04-26T05:06:09.278064+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:17:43.402733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282987+0000","last_change":"2024-04-26T05:03:50.868850+0000","last_active":"2024-04-26T05:06:09.282987+0000","last_peered":"2024-04-26T05:06:09.282987+0000","last_clean":"2024-04-26T05:06:09.282987+0000","last_became_active":"2024-04-26T05:03:50.868509+0000","last_became_peered":"2024-04-26T05:03:50.868509+0000","last_unstale":"2024-04-26T05:06:09.282987+0000","last_undegraded":"2024-04-26T05:06:09.282987+0000","last_fullsized":"2024-04-26T05:06:09.282987+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:03:08.728342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.291946+0000","last_change":"2024-04-26T05:06:10.291946+0000","last_active":"2024-04-26T05:06:10.291946+0000","last_peered":"2024-04-26T05:06:10.291946+0000","last_clean":"2024-04-26T05:06:10.291946+0000","last_became_active":"2024-04-26T05:06:10.291583+0000","last_became_peered":"2024-04-26T05:06:10.291583+0000","last_unstale":"2024-04-26T05:06:10.291946+0000","last_undegraded":"2024-04-26T05:06:10.291946+0000","last_fullsized":"2024-04-26T05:06:10.291946+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:52:57.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280637+0000","last_change":"2024-04-26T05:03:50.863193+0000","last_active":"2024-04-26T05:06:09.280637+0000","last_peered":"2024-04-26T05:06:09.280637+0000","last_clean":"2024-04-26T05:06:09.280637+0000","last_became_active":"2024-04-26T05:03:50.862811+0000","last_became_peered":"2024-04-26T05:03:50.862811+0000","last_unstale":"2024-04-26T05:06:09.280637+0000","last_undegraded":"2024-04-26T05:06:09.280637+0000","last_fullsized":"2024-04-26T05:06:09.280637+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:05:15.872947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294344+0000","last_change":"2024-04-26T05:06:10.293897+0000","last_active":"2024-04-26T05:06:10.294344+0000","last_peered":"2024-04-26T05:06:10.294344+0000","last_clean":"2024-04-26T05:06:10.294344+0000","last_became_active":"2024-04-26T05:06:10.293216+0000","last_became_peered":"2024-04-26T05:06:10.293216+0000","last_unstale":"2024-04-26T05:06:10.294344+0000","last_undegraded":"2024-04-26T05:06:10.294344+0000","last_fullsized":"2024-04-26T05:06:10.294344+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:15.516584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282111+0000","last_change":"2024-04-26T04:56:08.400160+0000","last_active":"2024-04-26T05:06:09.282111+0000","last_peered":"2024-04-26T05:06:09.282111+0000","last_clean":"2024-04-26T05:06:09.282111+0000","last_became_active":"2024-04-26T04:56:08.399849+0000","last_became_peered":"2024-04-26T04:56:08.399849+0000","last_unstale":"2024-04-26T05:06:09.282111+0000","last_undegraded":"2024-04-26T05:06:09.282111+0000","last_fullsized":"2024-04-26T05:06:09.282111+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:26:13.567428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288587+0000","last_change":"2024-04-26T05:06:10.288587+0000","last_active":"2024-04-26T05:06:10.288587+0000","last_peered":"2024-04-26T05:06:10.288587+0000","last_clean":"2024-04-26T05:06:10.288587+0000","last_became_active":"2024-04-26T05:06:10.288276+0000","last_became_peered":"2024-04-26T05:06:10.288276+0000","last_unstale":"2024-04-26T05:06:10.288587+0000","last_undegraded":"2024-04-26T05:06:10.288587+0000","last_fullsized":"2024-04-26T05:06:10.288587+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:43:59.434189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292394+0000","last_change":"2024-04-26T05:06:10.292394+0000","last_active":"2024-04-26T05:06:10.292394+0000","last_peered":"2024-04-26T05:06:10.292394+0000","last_clean":"2024-04-26T05:06:10.292394+0000","last_became_active":"2024-04-26T05:06:10.291719+0000","last_became_peered":"2024-04-26T05:06:10.291719+0000","last_unstale":"2024-04-26T05:06:10.292394+0000","last_undegraded":"2024-04-26T05:06:10.292394+0000","last_fullsized":"2024-04-26T05:06:10.292394+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:13:04.708586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292533+0000","last_change":"2024-04-26T05:06:10.292533+0000","last_active":"2024-04-26T05:06:10.292533+0000","last_peered":"2024-04-26T05:06:10.292533+0000","last_clean":"2024-04-26T05:06:10.292533+0000","last_became_active":"2024-04-26T05:06:10.291215+0000","last_became_peered":"2024-04-26T05:06:10.291215+0000","last_unstale":"2024-04-26T05:06:10.292533+0000","last_undegraded":"2024-04-26T05:06:10.292533+0000","last_fullsized":"2024-04-26T05:06:10.292533+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:16:15.561925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280266+0000","last_change":"2024-04-26T05:05:35.493408+0000","last_active":"2024-04-26T05:06:09.280266+0000","last_peered":"2024-04-26T05:06:09.280266+0000","last_clean":"2024-04-26T05:06:09.280266+0000","last_became_active":"2024-04-26T05:05:35.493021+0000","last_became_peered":"2024-04-26T05:05:35.493021+0000","last_unstale":"2024-04-26T05:06:09.280266+0000","last_undegraded":"2024-04-26T05:06:09.280266+0000","last_fullsized":"2024-04-26T05:06:09.280266+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:31.453096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275429+0000","last_change":"2024-04-26T05:05:01.306994+0000","last_active":"2024-04-26T05:06:09.275429+0000","last_peered":"2024-04-26T05:06:09.275429+0000","last_clean":"2024-04-26T05:06:09.275429+0000","last_became_active":"2024-04-26T05:05:01.306478+0000","last_became_peered":"2024-04-26T05:05:01.306478+0000","last_unstale":"2024-04-26T05:06:09.275429+0000","last_undegraded":"2024-04-26T05:06:09.275429+0000","last_fullsized":"2024-04-26T05:06:09.275429+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:19.828332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275380+0000","last_change":"2024-04-26T04:56:08.399546+0000","last_active":"2024-04-26T05:06:09.275380+0000","last_peered":"2024-04-26T05:06:09.275380+0000","last_clean":"2024-04-26T05:06:09.275380+0000","last_became_active":"2024-04-26T04:56:08.399266+0000","last_became_peered":"2024-04-26T04:56:08.399266+0000","last_unstale":"2024-04-26T05:06:09.275380+0000","last_undegraded":"2024-04-26T05:06:09.275380+0000","last_fullsized":"2024-04-26T05:06:09.275380+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:46:52.731455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.279888+0000","last_change":"2024-04-26T05:05:35.493759+0000","last_active":"2024-04-26T05:06:10.279888+0000","last_peered":"2024-04-26T05:06:10.279888+0000","last_clean":"2024-04-26T05:06:10.279888+0000","last_became_active":"2024-04-26T05:05:35.493446+0000","last_became_peered":"2024-04-26T05:05:35.493446+0000","last_unstale":"2024-04-26T05:06:10.279888+0000","last_undegraded":"2024-04-26T05:06:10.279888+0000","last_fullsized":"2024-04-26T05:06:10.279888+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:28:04.158907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289164+0000","last_change":"2024-04-26T05:06:10.289164+0000","last_active":"2024-04-26T05:06:10.289164+0000","last_peered":"2024-04-26T05:06:10.289164+0000","last_clean":"2024-04-26T05:06:10.289164+0000","last_became_active":"2024-04-26T05:06:10.288796+0000","last_became_peered":"2024-04-26T05:06:10.288796+0000","last_unstale":"2024-04-26T05:06:10.289164+0000","last_undegraded":"2024-04-26T05:06:10.289164+0000","last_fullsized":"2024-04-26T05:06:10.289164+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:40:08.343629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294639+0000","last_change":"2024-04-26T05:06:10.294639+0000","last_active":"2024-04-26T05:06:10.294639+0000","last_peered":"2024-04-26T05:06:10.294639+0000","last_clean":"2024-04-26T05:06:10.294639+0000","last_became_active":"2024-04-26T05:06:10.294193+0000","last_became_peered":"2024-04-26T05:06:10.294193+0000","last_unstale":"2024-04-26T05:06:10.294639+0000","last_undegraded":"2024-04-26T05:06:10.294639+0000","last_fullsized":"2024-04-26T05:06:10.294639+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:24:51.801620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278436+0000","last_change":"2024-04-26T05:05:01.306140+0000","last_active":"2024-04-26T05:06:09.278436+0000","last_peered":"2024-04-26T05:06:09.278436+0000","last_clean":"2024-04-26T05:06:09.278436+0000","last_became_active":"2024-04-26T05:05:01.305542+0000","last_became_peered":"2024-04-26T05:05:01.305542+0000","last_unstale":"2024-04-26T05:06:09.278436+0000","last_undegraded":"2024-04-26T05:06:09.278436+0000","last_fullsized":"2024-04-26T05:06:09.278436+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:40.640743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296220+0000","last_change":"2024-04-26T05:06:10.296220+0000","last_active":"2024-04-26T05:06:10.296220+0000","last_peered":"2024-04-26T05:06:10.296220+0000","last_clean":"2024-04-26T05:06:10.296220+0000","last_became_active":"2024-04-26T05:06:10.294923+0000","last_became_peered":"2024-04-26T05:06:10.294923+0000","last_unstale":"2024-04-26T05:06:10.296220+0000","last_undegraded":"2024-04-26T05:06:10.296220+0000","last_fullsized":"2024-04-26T05:06:10.296220+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:26:23.919695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.293058+0000","last_change":"2024-04-26T05:06:10.292656+0000","last_active":"2024-04-26T05:06:10.293058+0000","last_peered":"2024-04-26T05:06:10.293058+0000","last_clean":"2024-04-26T05:06:10.293058+0000","last_became_active":"2024-04-26T05:06:10.291777+0000","last_became_peered":"2024-04-26T05:06:10.291777+0000","last_unstale":"2024-04-26T05:06:10.293058+0000","last_undegraded":"2024-04-26T05:06:10.293058+0000","last_fullsized":"2024-04-26T05:06:10.293058+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:58:24.110953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.279918+0000","last_change":"2024-04-26T05:05:01.299539+0000","last_active":"2024-04-26T05:06:09.279918+0000","last_peered":"2024-04-26T05:06:09.279918+0000","last_clean":"2024-04-26T05:06:09.279918+0000","last_became_active":"2024-04-26T05:05:01.298988+0000","last_became_peered":"2024-04-26T05:05:01.298988+0000","last_unstale":"2024-04-26T05:06:09.279918+0000","last_undegraded":"2024-04-26T05:06:09.279918+0000","last_fullsized":"2024-04-26T05:06:09.279918+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:14:32.254680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296508+0000","last_change":"2024-04-26T05:06:10.296508+0000","last_active":"2024-04-26T05:06:10.296508+0000","last_peered":"2024-04-26T05:06:10.296508+0000","last_clean":"2024-04-26T05:06:10.296508+0000","last_became_active":"2024-04-26T05:06:10.295018+0000","last_became_peered":"2024-04-26T05:06:10.295018+0000","last_unstale":"2024-04-26T05:06:10.296508+0000","last_undegraded":"2024-04-26T05:06:10.296508+0000","last_fullsized":"2024-04-26T05:06:10.296508+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:19:18.698335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.293577+0000","last_change":"2024-04-26T05:06:10.293577+0000","last_active":"2024-04-26T05:06:10.293577+0000","last_peered":"2024-04-26T05:06:10.293577+0000","last_clean":"2024-04-26T05:06:10.293577+0000","last_became_active":"2024-04-26T05:06:10.293212+0000","last_became_peered":"2024-04-26T05:06:10.293212+0000","last_unstale":"2024-04-26T05:06:10.293577+0000","last_undegraded":"2024-04-26T05:06:10.293577+0000","last_fullsized":"2024-04-26T05:06:10.293577+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:04:10.927936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288260+0000","last_change":"2024-04-26T05:06:10.288260+0000","last_active":"2024-04-26T05:06:10.288260+0000","last_peered":"2024-04-26T05:06:10.288260+0000","last_clean":"2024-04-26T05:06:10.288260+0000","last_became_active":"2024-04-26T05:06:10.287923+0000","last_became_peered":"2024-04-26T05:06:10.287923+0000","last_unstale":"2024-04-26T05:06:10.288260+0000","last_undegraded":"2024-04-26T05:06:10.288260+0000","last_fullsized":"2024-04-26T05:06:10.288260+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:43:11.502894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294619+0000","last_change":"2024-04-26T05:06:10.294619+0000","last_active":"2024-04-26T05:06:10.294619+0000","last_peered":"2024-04-26T05:06:10.294619+0000","last_clean":"2024-04-26T05:06:10.294619+0000","last_became_active":"2024-04-26T05:06:10.294215+0000","last_became_peered":"2024-04-26T05:06:10.294215+0000","last_unstale":"2024-04-26T05:06:10.294619+0000","last_undegraded":"2024-04-26T05:06:10.294619+0000","last_fullsized":"2024-04-26T05:06:10.294619+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:48:28.705784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275170+0000","last_change":"2024-04-26T05:05:35.495708+0000","last_active":"2024-04-26T05:06:09.275170+0000","last_peered":"2024-04-26T05:06:09.275170+0000","last_clean":"2024-04-26T05:06:09.275170+0000","last_became_active":"2024-04-26T05:05:35.495401+0000","last_became_peered":"2024-04-26T05:05:35.495401+0000","last_unstale":"2024-04-26T05:06:09.275170+0000","last_undegraded":"2024-04-26T05:06:09.275170+0000","last_fullsized":"2024-04-26T05:06:09.275170+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:04:36.178002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18628,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":11984,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5319,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201437,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307340,"kb_used_data":5888,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93437812,"statfs":{"total":95995035648,"available":95680319488,"internally_reserved":0,"allocated":6029312,"data_stored":5423320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463075,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303244,"kb_used_data":5880,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93441908,"statfs":{"total":95995035648,"available":95684513792,"internally_reserved":0,"allocated":6021120,"data_stored":5423201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":100,"seq":429496729628,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32876,"kb_used_data":1184,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93712276,"statfs":{"total":95995035648,"available":95961370624,"internally_reserved":0,"allocated":1212416,"data_stored":633520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7683,"internal_metadata":32432637},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":95,"seq":408021893162,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33472,"kb_used_data":1776,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93711680,"statfs":{"total":95995035648,"available":95960760320,"internally_reserved":0,"allocated":1818624,"data_stored":1224999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8208,"internal_metadata":32432112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":105,"seq":450971566101,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32828,"kb_used_data":5288,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93712324,"statfs":{"total":95995035648,"available":95961419776,"internally_reserved":0,"allocated":5414912,"data_stored":4828320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8512,"internal_metadata":28171968},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":110,"seq":472446402574,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28724,"kb_used_data":1192,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716428,"statfs":{"total":95995035648,"available":95965622272,"internally_reserved":0,"allocated":1220608,"data_stored":634113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6360,"internal_metadata":28174120},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986348,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298568,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446584,"statfs":{"total":95995035648,"available":95689302016,"internally_reserved":0,"allocated":1236992,"data_stored":638636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724711,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":634397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3920,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1399,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6504,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5979,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2888,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3257,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:07:10.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:07:11.251 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:11.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:11 smithi195 ceph-mon[90222]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:11.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:11 smithi195 ceph-mon[90222]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:07:11.532 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:11 smithi152 ceph-mon[106538]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:11.532 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:11 smithi152 ceph-mon[106538]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:07:11.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:11 smithi152 ceph-mon[102000]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:11.532 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:11 smithi152 ceph-mon[102000]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:07:13.310 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:13.311 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-26T05:07:13.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:13 smithi152 ceph-mon[102000]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:13.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:13 smithi152 ceph-mon[106538]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:13.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:13 smithi195 ceph-mon[90222]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:14.037 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":245,"stamp":"2024-04-26T05:07:12.146725+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6494,"num_read_kb":12908,"num_write":1146,"num_write_kb":15039,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1335604,"kb_used_data":23612,"kb_used_omap":36,"kb_used_meta":1311835,"kb_avail":748625612,"statfs":{"total":767960285184,"available":766592626688,"internally_reserved":0,"allocated":24178688,"data_stored":19440506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37579,"internal_metadata":1343319349},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002775"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275046+0000","last_change":"2024-04-26T05:05:01.307763+0000","last_active":"2024-04-26T05:06:09.275046+0000","last_peered":"2024-04-26T05:06:09.275046+0000","last_clean":"2024-04-26T05:06:09.275046+0000","last_became_active":"2024-04-26T05:05:01.307169+0000","last_became_peered":"2024-04-26T05:05:01.307169+0000","last_unstale":"2024-04-26T05:06:09.275046+0000","last_undegraded":"2024-04-26T05:06:09.275046+0000","last_fullsized":"2024-04-26T05:06:09.275046+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:18:58.255124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":66,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283517+0000","last_change":"2024-04-26T05:05:35.493186+0000","last_active":"2024-04-26T05:06:10.283517+0000","last_peered":"2024-04-26T05:06:10.283517+0000","last_clean":"2024-04-26T05:06:10.283517+0000","last_became_active":"2024-04-26T05:05:35.492807+0000","last_became_peered":"2024-04-26T05:05:35.492807+0000","last_unstale":"2024-04-26T05:06:10.283517+0000","last_undegraded":"2024-04-26T05:06:10.283517+0000","last_fullsized":"2024-04-26T05:06:10.283517+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:26:40.440227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296310+0000","last_change":"2024-04-26T05:06:10.296310+0000","last_active":"2024-04-26T05:06:10.296310+0000","last_peered":"2024-04-26T05:06:10.296310+0000","last_clean":"2024-04-26T05:06:10.296310+0000","last_became_active":"2024-04-26T05:06:10.295934+0000","last_became_peered":"2024-04-26T05:06:10.295934+0000","last_unstale":"2024-04-26T05:06:10.296310+0000","last_undegraded":"2024-04-26T05:06:10.296310+0000","last_fullsized":"2024-04-26T05:06:10.296310+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:17:11.064327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":78,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294678+0000","last_change":"2024-04-26T05:06:10.294678+0000","last_active":"2024-04-26T05:06:10.294678+0000","last_peered":"2024-04-26T05:06:10.294678+0000","last_clean":"2024-04-26T05:06:10.294678+0000","last_became_active":"2024-04-26T05:06:10.294276+0000","last_became_peered":"2024-04-26T05:06:10.294276+0000","last_unstale":"2024-04-26T05:06:10.294678+0000","last_undegraded":"2024-04-26T05:06:10.294678+0000","last_fullsized":"2024-04-26T05:06:10.294678+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:34:06.294703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288581+0000","last_change":"2024-04-26T05:06:10.288581+0000","last_active":"2024-04-26T05:06:10.288581+0000","last_peered":"2024-04-26T05:06:10.288581+0000","last_clean":"2024-04-26T05:06:10.288581+0000","last_became_active":"2024-04-26T05:06:10.288223+0000","last_became_peered":"2024-04-26T05:06:10.288223+0000","last_unstale":"2024-04-26T05:06:10.288581+0000","last_undegraded":"2024-04-26T05:06:10.288581+0000","last_fullsized":"2024-04-26T05:06:10.288581+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:51:27.080918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280928+0000","last_change":"2024-04-26T04:56:08.400207+0000","last_active":"2024-04-26T05:06:09.280928+0000","last_peered":"2024-04-26T05:06:09.280928+0000","last_clean":"2024-04-26T05:06:09.280928+0000","last_became_active":"2024-04-26T04:56:08.399958+0000","last_became_peered":"2024-04-26T04:56:08.399958+0000","last_unstale":"2024-04-26T05:06:09.280928+0000","last_undegraded":"2024-04-26T05:06:09.280928+0000","last_fullsized":"2024-04-26T05:06:09.280928+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:58:07.756276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288545+0000","last_change":"2024-04-26T05:06:10.288545+0000","last_active":"2024-04-26T05:06:10.288545+0000","last_peered":"2024-04-26T05:06:10.288545+0000","last_clean":"2024-04-26T05:06:10.288545+0000","last_became_active":"2024-04-26T05:06:10.287834+0000","last_became_peered":"2024-04-26T05:06:10.287834+0000","last_unstale":"2024-04-26T05:06:10.288545+0000","last_undegraded":"2024-04-26T05:06:10.288545+0000","last_fullsized":"2024-04-26T05:06:10.288545+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:39:29.678078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278382+0000","last_change":"2024-04-26T05:05:01.306665+0000","last_active":"2024-04-26T05:06:09.278382+0000","last_peered":"2024-04-26T05:06:09.278382+0000","last_clean":"2024-04-26T05:06:09.278382+0000","last_became_active":"2024-04-26T05:05:01.306322+0000","last_became_peered":"2024-04-26T05:05:01.306322+0000","last_unstale":"2024-04-26T05:06:09.278382+0000","last_undegraded":"2024-04-26T05:06:09.278382+0000","last_fullsized":"2024-04-26T05:06:09.278382+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:34:24.045728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292007+0000","last_change":"2024-04-26T05:06:10.292007+0000","last_active":"2024-04-26T05:06:10.292007+0000","last_peered":"2024-04-26T05:06:10.292007+0000","last_clean":"2024-04-26T05:06:10.292007+0000","last_became_active":"2024-04-26T05:06:10.291651+0000","last_became_peered":"2024-04-26T05:06:10.291651+0000","last_unstale":"2024-04-26T05:06:10.292007+0000","last_undegraded":"2024-04-26T05:06:10.292007+0000","last_fullsized":"2024-04-26T05:06:10.292007+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:38:11.322567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":57,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.284072+0000","last_change":"2024-04-26T05:03:50.868029+0000","last_active":"2024-04-26T05:06:10.284072+0000","last_peered":"2024-04-26T05:06:10.284072+0000","last_clean":"2024-04-26T05:06:10.284072+0000","last_became_active":"2024-04-26T05:03:50.867549+0000","last_became_peered":"2024-04-26T05:03:50.867549+0000","last_unstale":"2024-04-26T05:06:10.284072+0000","last_undegraded":"2024-04-26T05:06:10.284072+0000","last_fullsized":"2024-04-26T05:06:10.284072+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:21:59.272714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.277125+0000","last_change":"2024-04-26T04:56:08.417990+0000","last_active":"2024-04-26T05:06:09.277125+0000","last_peered":"2024-04-26T05:06:09.277125+0000","last_clean":"2024-04-26T05:06:09.277125+0000","last_became_active":"2024-04-26T04:56:08.417703+0000","last_became_peered":"2024-04-26T04:56:08.417703+0000","last_unstale":"2024-04-26T05:06:09.277125+0000","last_undegraded":"2024-04-26T05:06:09.277125+0000","last_fullsized":"2024-04-26T05:06:09.277125+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:49:52.041048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":66,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283769+0000","last_change":"2024-04-26T05:05:01.307393+0000","last_active":"2024-04-26T05:06:10.283769+0000","last_peered":"2024-04-26T05:06:10.283769+0000","last_clean":"2024-04-26T05:06:10.283769+0000","last_became_active":"2024-04-26T05:05:01.307007+0000","last_became_peered":"2024-04-26T05:05:01.307007+0000","last_unstale":"2024-04-26T05:06:10.283769+0000","last_undegraded":"2024-04-26T05:06:10.283769+0000","last_fullsized":"2024-04-26T05:06:10.283769+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:51:50.722261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"87'97","reported_seq":441,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.279971+0000","last_change":"2024-04-26T05:05:35.492969+0000","last_active":"2024-04-26T05:06:09.279971+0000","last_peered":"2024-04-26T05:06:09.279971+0000","last_clean":"2024-04-26T05:06:09.279971+0000","last_became_active":"2024-04-26T05:05:35.492441+0000","last_became_peered":"2024-04-26T05:05:35.492441+0000","last_unstale":"2024-04-26T05:06:09.279971+0000","last_undegraded":"2024-04-26T05:06:09.279971+0000","last_fullsized":"2024-04-26T05:06:09.279971+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:45:40.923654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00087697999999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":696,"num_read_kb":866,"num_write":90,"num_write_kb":5299,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296523+0000","last_change":"2024-04-26T05:06:10.296523+0000","last_active":"2024-04-26T05:06:10.296523+0000","last_peered":"2024-04-26T05:06:10.296523+0000","last_clean":"2024-04-26T05:06:10.296523+0000","last_became_active":"2024-04-26T05:06:10.296132+0000","last_became_peered":"2024-04-26T05:06:10.296132+0000","last_unstale":"2024-04-26T05:06:10.296523+0000","last_undegraded":"2024-04-26T05:06:10.296523+0000","last_fullsized":"2024-04-26T05:06:10.296523+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:41.356448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"87'105","reported_seq":320,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280655+0000","last_change":"2024-04-26T05:05:01.303861+0000","last_active":"2024-04-26T05:06:10.280655+0000","last_peered":"2024-04-26T05:06:10.280655+0000","last_clean":"2024-04-26T05:06:10.280655+0000","last_became_active":"2024-04-26T05:05:01.303177+0000","last_became_peered":"2024-04-26T05:05:01.303177+0000","last_unstale":"2024-04-26T05:06:10.280655+0000","last_undegraded":"2024-04-26T05:06:10.280655+0000","last_fullsized":"2024-04-26T05:06:10.280655+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:58:38.326483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022107600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":369,"num_read_kb":388,"num_write":84,"num_write_kb":1127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296560+0000","last_change":"2024-04-26T05:06:10.296560+0000","last_active":"2024-04-26T05:06:10.296560+0000","last_peered":"2024-04-26T05:06:10.296560+0000","last_clean":"2024-04-26T05:06:10.296560+0000","last_became_active":"2024-04-26T05:06:10.296000+0000","last_became_peered":"2024-04-26T05:06:10.296000+0000","last_unstale":"2024-04-26T05:06:10.296560+0000","last_undegraded":"2024-04-26T05:06:10.296560+0000","last_fullsized":"2024-04-26T05:06:10.296560+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:17:30.714197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283829+0000","last_change":"2024-04-26T05:03:50.865358+0000","last_active":"2024-04-26T05:06:10.283829+0000","last_peered":"2024-04-26T05:06:10.283829+0000","last_clean":"2024-04-26T05:06:10.283829+0000","last_became_active":"2024-04-26T05:03:50.864950+0000","last_became_peered":"2024-04-26T05:03:50.864950+0000","last_unstale":"2024-04-26T05:06:10.283829+0000","last_undegraded":"2024-04-26T05:06:10.283829+0000","last_fullsized":"2024-04-26T05:06:10.283829+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:57:40.431639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280681+0000","last_change":"2024-04-26T05:05:35.494290+0000","last_active":"2024-04-26T05:06:10.280681+0000","last_peered":"2024-04-26T05:06:10.280681+0000","last_clean":"2024-04-26T05:06:10.280681+0000","last_became_active":"2024-04-26T05:05:35.493789+0000","last_became_peered":"2024-04-26T05:05:35.493789+0000","last_unstale":"2024-04-26T05:06:10.280681+0000","last_undegraded":"2024-04-26T05:06:10.280681+0000","last_fullsized":"2024-04-26T05:06:10.280681+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:46:00.079348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"87'36","reported_seq":247,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280434+0000","last_change":"2024-04-26T05:03:50.870038+0000","last_active":"2024-04-26T05:06:09.280434+0000","last_peered":"2024-04-26T05:06:09.280434+0000","last_clean":"2024-04-26T05:06:09.280434+0000","last_became_active":"2024-04-26T05:03:50.869602+0000","last_became_peered":"2024-04-26T05:03:50.869602+0000","last_unstale":"2024-04-26T05:06:09.280434+0000","last_undegraded":"2024-04-26T05:06:09.280434+0000","last_fullsized":"2024-04-26T05:06:09.280434+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":36,"log_dups_size":0,"ondisk_log_size":36,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:44:48.772118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031756800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":214,"num_read_kb":1209,"num_write":25,"num_write_kb":1044,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283916+0000","last_change":"2024-04-26T05:05:35.493573+0000","last_active":"2024-04-26T05:06:10.283916+0000","last_peered":"2024-04-26T05:06:10.283916+0000","last_clean":"2024-04-26T05:06:10.283916+0000","last_became_active":"2024-04-26T05:05:35.493241+0000","last_became_peered":"2024-04-26T05:05:35.493241+0000","last_unstale":"2024-04-26T05:06:10.283916+0000","last_undegraded":"2024-04-26T05:06:10.283916+0000","last_fullsized":"2024-04-26T05:06:10.283916+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:33.340797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292519+0000","last_change":"2024-04-26T05:06:10.292519+0000","last_active":"2024-04-26T05:06:10.292519+0000","last_peered":"2024-04-26T05:06:10.292519+0000","last_clean":"2024-04-26T05:06:10.292519+0000","last_became_active":"2024-04-26T05:06:10.292205+0000","last_became_peered":"2024-04-26T05:06:10.292205+0000","last_unstale":"2024-04-26T05:06:10.292519+0000","last_undegraded":"2024-04-26T05:06:10.292519+0000","last_fullsized":"2024-04-26T05:06:10.292519+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:48:57.035079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"87'88","reported_seq":436,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282363+0000","last_change":"2024-04-26T05:05:01.305251+0000","last_active":"2024-04-26T05:06:09.282363+0000","last_peered":"2024-04-26T05:06:09.282363+0000","last_clean":"2024-04-26T05:06:09.282363+0000","last_became_active":"2024-04-26T05:05:01.304749+0000","last_became_peered":"2024-04-26T05:05:01.304749+0000","last_unstale":"2024-04-26T05:06:09.282363+0000","last_undegraded":"2024-04-26T05:06:09.282363+0000","last_fullsized":"2024-04-26T05:06:09.282363+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:14:35.322089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038853800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":593,"num_read_kb":510,"num_write":72,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282133+0000","last_change":"2024-04-26T04:56:08.735555+0000","last_active":"2024-04-26T05:06:09.282133+0000","last_peered":"2024-04-26T05:06:09.282133+0000","last_clean":"2024-04-26T05:06:09.282133+0000","last_became_active":"2024-04-26T04:56:08.735298+0000","last_became_peered":"2024-04-26T04:56:08.735298+0000","last_unstale":"2024-04-26T05:06:09.282133+0000","last_undegraded":"2024-04-26T05:06:09.282133+0000","last_fullsized":"2024-04-26T05:06:09.282133+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:08.329803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294348+0000","last_change":"2024-04-26T05:06:10.294348+0000","last_active":"2024-04-26T05:06:10.294348+0000","last_peered":"2024-04-26T05:06:10.294348+0000","last_clean":"2024-04-26T05:06:10.294348+0000","last_became_active":"2024-04-26T05:06:10.294038+0000","last_became_peered":"2024-04-26T05:06:10.294038+0000","last_unstale":"2024-04-26T05:06:10.294348+0000","last_undegraded":"2024-04-26T05:06:10.294348+0000","last_fullsized":"2024-04-26T05:06:10.294348+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:12.400548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":450,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282266+0000","last_change":"2024-04-26T05:03:50.869031+0000","last_active":"2024-04-26T05:06:09.282266+0000","last_peered":"2024-04-26T05:06:09.282266+0000","last_clean":"2024-04-26T05:06:09.282266+0000","last_became_active":"2024-04-26T05:03:50.868240+0000","last_became_peered":"2024-04-26T05:03:50.868240+0000","last_unstale":"2024-04-26T05:06:09.282266+0000","last_undegraded":"2024-04-26T05:06:09.282266+0000","last_fullsized":"2024-04-26T05:06:09.282266+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:52:21.376988+0000","last_clean_scrub_stamp":"2024-04-26T04:52:21.376988+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:51:08.168877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"87'216","reported_seq":760,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278214+0000","last_change":"2024-04-26T05:05:35.493996+0000","last_active":"2024-04-26T05:06:09.278214+0000","last_peered":"2024-04-26T05:06:09.278214+0000","last_clean":"2024-04-26T05:06:09.278214+0000","last_became_active":"2024-04-26T05:05:35.493332+0000","last_became_peered":"2024-04-26T05:05:35.493332+0000","last_unstale":"2024-04-26T05:06:09.278214+0000","last_undegraded":"2024-04-26T05:06:09.278214+0000","last_fullsized":"2024-04-26T05:06:09.278214+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":216,"log_dups_size":0,"ondisk_log_size":216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:03:37.684836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000304054,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1035,"num_read_kb":923,"num_write":190,"num_write_kb":120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280224+0000","last_change":"2024-04-26T05:05:01.306363+0000","last_active":"2024-04-26T05:06:10.280224+0000","last_peered":"2024-04-26T05:06:10.280224+0000","last_clean":"2024-04-26T05:06:10.280224+0000","last_became_active":"2024-04-26T05:05:01.304618+0000","last_became_peered":"2024-04-26T05:05:01.304618+0000","last_unstale":"2024-04-26T05:06:10.280224+0000","last_undegraded":"2024-04-26T05:06:10.280224+0000","last_fullsized":"2024-04-26T05:06:10.280224+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:17.531286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280076+0000","last_change":"2024-04-26T05:05:35.492906+0000","last_active":"2024-04-26T05:06:10.280076+0000","last_peered":"2024-04-26T05:06:10.280076+0000","last_clean":"2024-04-26T05:06:10.280076+0000","last_became_active":"2024-04-26T05:05:35.492594+0000","last_became_peered":"2024-04-26T05:05:35.492594+0000","last_unstale":"2024-04-26T05:06:10.280076+0000","last_undegraded":"2024-04-26T05:06:10.280076+0000","last_fullsized":"2024-04-26T05:06:10.280076+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:14:12.268165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"87'190","reported_seq":825,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.293037+0000","last_change":"2024-04-26T05:06:10.293037+0000","last_active":"2024-04-26T05:06:10.293037+0000","last_peered":"2024-04-26T05:06:10.293037+0000","last_clean":"2024-04-26T05:06:10.293037+0000","last_became_active":"2024-04-26T05:06:10.292403+0000","last_became_peered":"2024-04-26T05:06:10.292403+0000","last_unstale":"2024-04-26T05:06:10.293037+0000","last_undegraded":"2024-04-26T05:06:10.293037+0000","last_fullsized":"2024-04-26T05:06:10.293037+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:46.240751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000392913,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1328,"num_read_kb":3434,"num_write":163,"num_write_kb":2297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289550+0000","last_change":"2024-04-26T05:06:10.288557+0000","last_active":"2024-04-26T05:06:10.289550+0000","last_peered":"2024-04-26T05:06:10.289550+0000","last_clean":"2024-04-26T05:06:10.289550+0000","last_became_active":"2024-04-26T05:06:10.287948+0000","last_became_peered":"2024-04-26T05:06:10.287948+0000","last_unstale":"2024-04-26T05:06:10.289550+0000","last_undegraded":"2024-04-26T05:06:10.289550+0000","last_fullsized":"2024-04-26T05:06:10.289550+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:09:22.728442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275314+0000","last_change":"2024-04-26T05:03:50.866118+0000","last_active":"2024-04-26T05:06:09.275314+0000","last_peered":"2024-04-26T05:06:09.275314+0000","last_clean":"2024-04-26T05:06:09.275314+0000","last_became_active":"2024-04-26T05:03:50.865744+0000","last_became_peered":"2024-04-26T05:03:50.865744+0000","last_unstale":"2024-04-26T05:06:09.275314+0000","last_undegraded":"2024-04-26T05:06:09.275314+0000","last_fullsized":"2024-04-26T05:06:09.275314+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:35:07.013499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"87'77","reported_seq":557,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278133+0000","last_change":"2024-04-26T05:05:01.306063+0000","last_active":"2024-04-26T05:06:09.278133+0000","last_peered":"2024-04-26T05:06:09.278133+0000","last_clean":"2024-04-26T05:06:09.278133+0000","last_became_active":"2024-04-26T05:05:01.305595+0000","last_became_peered":"2024-04-26T05:05:01.305595+0000","last_unstale":"2024-04-26T05:06:09.278133+0000","last_undegraded":"2024-04-26T05:06:09.278133+0000","last_fullsized":"2024-04-26T05:06:09.278133+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":77,"log_dups_size":0,"ondisk_log_size":77,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:24:24.544418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000660861,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":598,"num_read_kb":2063,"num_write":68,"num_write_kb":1301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283265+0000","last_change":"2024-04-26T05:05:35.497639+0000","last_active":"2024-04-26T05:06:10.283265+0000","last_peered":"2024-04-26T05:06:10.283265+0000","last_clean":"2024-04-26T05:06:10.283265+0000","last_became_active":"2024-04-26T05:05:35.496820+0000","last_became_peered":"2024-04-26T05:05:35.496820+0000","last_unstale":"2024-04-26T05:06:10.283265+0000","last_undegraded":"2024-04-26T05:06:10.283265+0000","last_fullsized":"2024-04-26T05:06:10.283265+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:12:15.241870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289553+0000","last_change":"2024-04-26T05:06:10.289553+0000","last_active":"2024-04-26T05:06:10.289553+0000","last_peered":"2024-04-26T05:06:10.289553+0000","last_clean":"2024-04-26T05:06:10.289553+0000","last_became_active":"2024-04-26T05:06:10.289218+0000","last_became_peered":"2024-04-26T05:06:10.289218+0000","last_unstale":"2024-04-26T05:06:10.289553+0000","last_undegraded":"2024-04-26T05:06:10.289553+0000","last_fullsized":"2024-04-26T05:06:10.289553+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:49:07.184501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"87'167","reported_seq":764,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.280749+0000","last_change":"2024-04-26T05:05:01.307161+0000","last_active":"2024-04-26T05:06:10.280749+0000","last_peered":"2024-04-26T05:06:10.280749+0000","last_clean":"2024-04-26T05:06:10.280749+0000","last_became_active":"2024-04-26T05:05:01.306323+0000","last_became_peered":"2024-04-26T05:05:01.306323+0000","last_unstale":"2024-04-26T05:06:10.280749+0000","last_undegraded":"2024-04-26T05:06:10.280749+0000","last_fullsized":"2024-04-26T05:06:10.280749+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:54:54.283157+0000","last_clean_scrub_stamp":"2024-04-26T04:54:54.283157+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:17:40.461307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050161400000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1161,"num_read_kb":2591,"num_write":154,"num_write_kb":1373,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289869+0000","last_change":"2024-04-26T05:06:10.289869+0000","last_active":"2024-04-26T05:06:10.289869+0000","last_peered":"2024-04-26T05:06:10.289869+0000","last_clean":"2024-04-26T05:06:10.289869+0000","last_became_active":"2024-04-26T05:06:10.289551+0000","last_became_peered":"2024-04-26T05:06:10.289551+0000","last_unstale":"2024-04-26T05:06:10.289869+0000","last_undegraded":"2024-04-26T05:06:10.289869+0000","last_fullsized":"2024-04-26T05:06:10.289869+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:52:04.722533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292244+0000","last_change":"2024-04-26T05:06:10.292244+0000","last_active":"2024-04-26T05:06:10.292244+0000","last_peered":"2024-04-26T05:06:10.292244+0000","last_clean":"2024-04-26T05:06:10.292244+0000","last_became_active":"2024-04-26T05:06:10.291108+0000","last_became_peered":"2024-04-26T05:06:10.291108+0000","last_unstale":"2024-04-26T05:06:10.292244+0000","last_undegraded":"2024-04-26T05:06:10.292244+0000","last_fullsized":"2024-04-26T05:06:10.292244+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:47:16.984760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.276134+0000","last_change":"2024-04-26T05:05:01.309433+0000","last_active":"2024-04-26T05:06:09.276134+0000","last_peered":"2024-04-26T05:06:09.276134+0000","last_clean":"2024-04-26T05:06:09.276134+0000","last_became_active":"2024-04-26T05:05:01.308770+0000","last_became_peered":"2024-04-26T05:05:01.308770+0000","last_unstale":"2024-04-26T05:06:09.276134+0000","last_undegraded":"2024-04-26T05:06:09.276134+0000","last_fullsized":"2024-04-26T05:06:09.276134+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:34:58.585785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.277209+0000","last_change":"2024-04-26T05:05:35.497619+0000","last_active":"2024-04-26T05:06:09.277209+0000","last_peered":"2024-04-26T05:06:09.277209+0000","last_clean":"2024-04-26T05:06:09.277209+0000","last_became_active":"2024-04-26T05:05:35.497260+0000","last_became_peered":"2024-04-26T05:05:35.497260+0000","last_unstale":"2024-04-26T05:06:09.277209+0000","last_undegraded":"2024-04-26T05:06:09.277209+0000","last_fullsized":"2024-04-26T05:06:09.277209+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:29:28.499889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.276687+0000","last_change":"2024-04-26T05:05:35.495597+0000","last_active":"2024-04-26T05:06:09.276687+0000","last_peered":"2024-04-26T05:06:09.276687+0000","last_clean":"2024-04-26T05:06:09.276687+0000","last_became_active":"2024-04-26T05:05:35.495274+0000","last_became_peered":"2024-04-26T05:05:35.495274+0000","last_unstale":"2024-04-26T05:06:09.276687+0000","last_undegraded":"2024-04-26T05:06:09.276687+0000","last_fullsized":"2024-04-26T05:06:09.276687+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:16.648302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280880+0000","last_change":"2024-04-26T05:05:01.301531+0000","last_active":"2024-04-26T05:06:09.280880+0000","last_peered":"2024-04-26T05:06:09.280880+0000","last_clean":"2024-04-26T05:06:09.280880+0000","last_became_active":"2024-04-26T05:05:01.301149+0000","last_became_peered":"2024-04-26T05:05:01.301149+0000","last_unstale":"2024-04-26T05:06:09.280880+0000","last_undegraded":"2024-04-26T05:06:09.280880+0000","last_fullsized":"2024-04-26T05:06:09.280880+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:49.906225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.291847+0000","last_change":"2024-04-26T05:06:10.291847+0000","last_active":"2024-04-26T05:06:10.291847+0000","last_peered":"2024-04-26T05:06:10.291847+0000","last_clean":"2024-04-26T05:06:10.291847+0000","last_became_active":"2024-04-26T05:06:10.291495+0000","last_became_peered":"2024-04-26T05:06:10.291495+0000","last_unstale":"2024-04-26T05:06:10.291847+0000","last_undegraded":"2024-04-26T05:06:10.291847+0000","last_fullsized":"2024-04-26T05:06:10.291847+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:44.938298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296035+0000","last_change":"2024-04-26T05:06:10.296035+0000","last_active":"2024-04-26T05:06:10.296035+0000","last_peered":"2024-04-26T05:06:10.296035+0000","last_clean":"2024-04-26T05:06:10.296035+0000","last_became_active":"2024-04-26T05:06:10.294898+0000","last_became_peered":"2024-04-26T05:06:10.294898+0000","last_unstale":"2024-04-26T05:06:10.296035+0000","last_undegraded":"2024-04-26T05:06:10.296035+0000","last_fullsized":"2024-04-26T05:06:10.296035+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:18:36.704546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.283942+0000","last_change":"2024-04-26T05:05:01.305384+0000","last_active":"2024-04-26T05:06:10.283942+0000","last_peered":"2024-04-26T05:06:10.283942+0000","last_clean":"2024-04-26T05:06:10.283942+0000","last_became_active":"2024-04-26T05:05:01.305046+0000","last_became_peered":"2024-04-26T05:05:01.305046+0000","last_unstale":"2024-04-26T05:06:10.283942+0000","last_undegraded":"2024-04-26T05:06:10.283942+0000","last_fullsized":"2024-04-26T05:06:10.283942+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:26.099419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.291825+0000","last_change":"2024-04-26T05:06:10.291825+0000","last_active":"2024-04-26T05:06:10.291825+0000","last_peered":"2024-04-26T05:06:10.291825+0000","last_clean":"2024-04-26T05:06:10.291825+0000","last_became_active":"2024-04-26T05:06:10.291537+0000","last_became_peered":"2024-04-26T05:06:10.291537+0000","last_unstale":"2024-04-26T05:06:10.291825+0000","last_undegraded":"2024-04-26T05:06:10.291825+0000","last_fullsized":"2024-04-26T05:06:10.291825+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:56:13.081732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280736+0000","last_change":"2024-04-26T05:05:01.299480+0000","last_active":"2024-04-26T05:06:09.280736+0000","last_peered":"2024-04-26T05:06:09.280736+0000","last_clean":"2024-04-26T05:06:09.280736+0000","last_became_active":"2024-04-26T05:05:01.299142+0000","last_became_peered":"2024-04-26T05:05:01.299142+0000","last_unstale":"2024-04-26T05:06:09.280736+0000","last_undegraded":"2024-04-26T05:06:09.280736+0000","last_fullsized":"2024-04-26T05:06:09.280736+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:37:49.299492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280531+0000","last_change":"2024-04-26T04:56:08.400274+0000","last_active":"2024-04-26T05:06:09.280531+0000","last_peered":"2024-04-26T05:06:09.280531+0000","last_clean":"2024-04-26T05:06:09.280531+0000","last_became_active":"2024-04-26T04:56:08.400012+0000","last_became_peered":"2024-04-26T04:56:08.400012+0000","last_unstale":"2024-04-26T05:06:09.280531+0000","last_undegraded":"2024-04-26T05:06:09.280531+0000","last_fullsized":"2024-04-26T05:06:09.280531+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:06:01.413415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288328+0000","last_change":"2024-04-26T05:06:10.288328+0000","last_active":"2024-04-26T05:06:10.288328+0000","last_peered":"2024-04-26T05:06:10.288328+0000","last_clean":"2024-04-26T05:06:10.288328+0000","last_became_active":"2024-04-26T05:06:10.287876+0000","last_became_peered":"2024-04-26T05:06:10.287876+0000","last_unstale":"2024-04-26T05:06:10.288328+0000","last_undegraded":"2024-04-26T05:06:10.288328+0000","last_fullsized":"2024-04-26T05:06:10.288328+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:46:13.556328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280231+0000","last_change":"2024-04-26T05:05:01.305251+0000","last_active":"2024-04-26T05:06:09.280231+0000","last_peered":"2024-04-26T05:06:09.280231+0000","last_clean":"2024-04-26T05:06:09.280231+0000","last_became_active":"2024-04-26T05:05:01.304784+0000","last_became_peered":"2024-04-26T05:05:01.304784+0000","last_unstale":"2024-04-26T05:06:09.280231+0000","last_undegraded":"2024-04-26T05:06:09.280231+0000","last_fullsized":"2024-04-26T05:06:09.280231+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:58:25.361117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278064+0000","last_change":"2024-04-26T04:56:08.735825+0000","last_active":"2024-04-26T05:06:09.278064+0000","last_peered":"2024-04-26T05:06:09.278064+0000","last_clean":"2024-04-26T05:06:09.278064+0000","last_became_active":"2024-04-26T04:56:08.735226+0000","last_became_peered":"2024-04-26T04:56:08.735226+0000","last_unstale":"2024-04-26T05:06:09.278064+0000","last_undegraded":"2024-04-26T05:06:09.278064+0000","last_fullsized":"2024-04-26T05:06:09.278064+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:17:43.402733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282987+0000","last_change":"2024-04-26T05:03:50.868850+0000","last_active":"2024-04-26T05:06:09.282987+0000","last_peered":"2024-04-26T05:06:09.282987+0000","last_clean":"2024-04-26T05:06:09.282987+0000","last_became_active":"2024-04-26T05:03:50.868509+0000","last_became_peered":"2024-04-26T05:03:50.868509+0000","last_unstale":"2024-04-26T05:06:09.282987+0000","last_undegraded":"2024-04-26T05:06:09.282987+0000","last_fullsized":"2024-04-26T05:06:09.282987+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:03:08.728342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.291946+0000","last_change":"2024-04-26T05:06:10.291946+0000","last_active":"2024-04-26T05:06:10.291946+0000","last_peered":"2024-04-26T05:06:10.291946+0000","last_clean":"2024-04-26T05:06:10.291946+0000","last_became_active":"2024-04-26T05:06:10.291583+0000","last_became_peered":"2024-04-26T05:06:10.291583+0000","last_unstale":"2024-04-26T05:06:10.291946+0000","last_undegraded":"2024-04-26T05:06:10.291946+0000","last_fullsized":"2024-04-26T05:06:10.291946+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:52:57.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280637+0000","last_change":"2024-04-26T05:03:50.863193+0000","last_active":"2024-04-26T05:06:09.280637+0000","last_peered":"2024-04-26T05:06:09.280637+0000","last_clean":"2024-04-26T05:06:09.280637+0000","last_became_active":"2024-04-26T05:03:50.862811+0000","last_became_peered":"2024-04-26T05:03:50.862811+0000","last_unstale":"2024-04-26T05:06:09.280637+0000","last_undegraded":"2024-04-26T05:06:09.280637+0000","last_fullsized":"2024-04-26T05:06:09.280637+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:05:15.872947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294344+0000","last_change":"2024-04-26T05:06:10.293897+0000","last_active":"2024-04-26T05:06:10.294344+0000","last_peered":"2024-04-26T05:06:10.294344+0000","last_clean":"2024-04-26T05:06:10.294344+0000","last_became_active":"2024-04-26T05:06:10.293216+0000","last_became_peered":"2024-04-26T05:06:10.293216+0000","last_unstale":"2024-04-26T05:06:10.294344+0000","last_undegraded":"2024-04-26T05:06:10.294344+0000","last_fullsized":"2024-04-26T05:06:10.294344+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:15.516584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.282111+0000","last_change":"2024-04-26T04:56:08.400160+0000","last_active":"2024-04-26T05:06:09.282111+0000","last_peered":"2024-04-26T05:06:09.282111+0000","last_clean":"2024-04-26T05:06:09.282111+0000","last_became_active":"2024-04-26T04:56:08.399849+0000","last_became_peered":"2024-04-26T04:56:08.399849+0000","last_unstale":"2024-04-26T05:06:09.282111+0000","last_undegraded":"2024-04-26T05:06:09.282111+0000","last_fullsized":"2024-04-26T05:06:09.282111+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:26:13.567428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288587+0000","last_change":"2024-04-26T05:06:10.288587+0000","last_active":"2024-04-26T05:06:10.288587+0000","last_peered":"2024-04-26T05:06:10.288587+0000","last_clean":"2024-04-26T05:06:10.288587+0000","last_became_active":"2024-04-26T05:06:10.288276+0000","last_became_peered":"2024-04-26T05:06:10.288276+0000","last_unstale":"2024-04-26T05:06:10.288587+0000","last_undegraded":"2024-04-26T05:06:10.288587+0000","last_fullsized":"2024-04-26T05:06:10.288587+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:43:59.434189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292394+0000","last_change":"2024-04-26T05:06:10.292394+0000","last_active":"2024-04-26T05:06:10.292394+0000","last_peered":"2024-04-26T05:06:10.292394+0000","last_clean":"2024-04-26T05:06:10.292394+0000","last_became_active":"2024-04-26T05:06:10.291719+0000","last_became_peered":"2024-04-26T05:06:10.291719+0000","last_unstale":"2024-04-26T05:06:10.292394+0000","last_undegraded":"2024-04-26T05:06:10.292394+0000","last_fullsized":"2024-04-26T05:06:10.292394+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:13:04.708586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.292533+0000","last_change":"2024-04-26T05:06:10.292533+0000","last_active":"2024-04-26T05:06:10.292533+0000","last_peered":"2024-04-26T05:06:10.292533+0000","last_clean":"2024-04-26T05:06:10.292533+0000","last_became_active":"2024-04-26T05:06:10.291215+0000","last_became_peered":"2024-04-26T05:06:10.291215+0000","last_unstale":"2024-04-26T05:06:10.292533+0000","last_undegraded":"2024-04-26T05:06:10.292533+0000","last_fullsized":"2024-04-26T05:06:10.292533+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:16:15.561925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.280266+0000","last_change":"2024-04-26T05:05:35.493408+0000","last_active":"2024-04-26T05:06:09.280266+0000","last_peered":"2024-04-26T05:06:09.280266+0000","last_clean":"2024-04-26T05:06:09.280266+0000","last_became_active":"2024-04-26T05:05:35.493021+0000","last_became_peered":"2024-04-26T05:05:35.493021+0000","last_unstale":"2024-04-26T05:06:09.280266+0000","last_undegraded":"2024-04-26T05:06:09.280266+0000","last_fullsized":"2024-04-26T05:06:09.280266+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:31.453096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275429+0000","last_change":"2024-04-26T05:05:01.306994+0000","last_active":"2024-04-26T05:06:09.275429+0000","last_peered":"2024-04-26T05:06:09.275429+0000","last_clean":"2024-04-26T05:06:09.275429+0000","last_became_active":"2024-04-26T05:05:01.306478+0000","last_became_peered":"2024-04-26T05:05:01.306478+0000","last_unstale":"2024-04-26T05:06:09.275429+0000","last_undegraded":"2024-04-26T05:06:09.275429+0000","last_fullsized":"2024-04-26T05:06:09.275429+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:19.828332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275380+0000","last_change":"2024-04-26T04:56:08.399546+0000","last_active":"2024-04-26T05:06:09.275380+0000","last_peered":"2024-04-26T05:06:09.275380+0000","last_clean":"2024-04-26T05:06:09.275380+0000","last_became_active":"2024-04-26T04:56:08.399266+0000","last_became_peered":"2024-04-26T04:56:08.399266+0000","last_unstale":"2024-04-26T05:06:09.275380+0000","last_undegraded":"2024-04-26T05:06:09.275380+0000","last_fullsized":"2024-04-26T05:06:09.275380+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:46:52.731455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.279888+0000","last_change":"2024-04-26T05:05:35.493759+0000","last_active":"2024-04-26T05:06:10.279888+0000","last_peered":"2024-04-26T05:06:10.279888+0000","last_clean":"2024-04-26T05:06:10.279888+0000","last_became_active":"2024-04-26T05:05:35.493446+0000","last_became_peered":"2024-04-26T05:05:35.493446+0000","last_unstale":"2024-04-26T05:06:10.279888+0000","last_undegraded":"2024-04-26T05:06:10.279888+0000","last_fullsized":"2024-04-26T05:06:10.279888+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:28:04.158907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.289164+0000","last_change":"2024-04-26T05:06:10.289164+0000","last_active":"2024-04-26T05:06:10.289164+0000","last_peered":"2024-04-26T05:06:10.289164+0000","last_clean":"2024-04-26T05:06:10.289164+0000","last_became_active":"2024-04-26T05:06:10.288796+0000","last_became_peered":"2024-04-26T05:06:10.288796+0000","last_unstale":"2024-04-26T05:06:10.289164+0000","last_undegraded":"2024-04-26T05:06:10.289164+0000","last_fullsized":"2024-04-26T05:06:10.289164+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:40:08.343629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294639+0000","last_change":"2024-04-26T05:06:10.294639+0000","last_active":"2024-04-26T05:06:10.294639+0000","last_peered":"2024-04-26T05:06:10.294639+0000","last_clean":"2024-04-26T05:06:10.294639+0000","last_became_active":"2024-04-26T05:06:10.294193+0000","last_became_peered":"2024-04-26T05:06:10.294193+0000","last_unstale":"2024-04-26T05:06:10.294639+0000","last_undegraded":"2024-04-26T05:06:10.294639+0000","last_fullsized":"2024-04-26T05:06:10.294639+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:24:51.801620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.278436+0000","last_change":"2024-04-26T05:05:01.306140+0000","last_active":"2024-04-26T05:06:09.278436+0000","last_peered":"2024-04-26T05:06:09.278436+0000","last_clean":"2024-04-26T05:06:09.278436+0000","last_became_active":"2024-04-26T05:05:01.305542+0000","last_became_peered":"2024-04-26T05:05:01.305542+0000","last_unstale":"2024-04-26T05:06:09.278436+0000","last_undegraded":"2024-04-26T05:06:09.278436+0000","last_fullsized":"2024-04-26T05:06:09.278436+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:40.640743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296220+0000","last_change":"2024-04-26T05:06:10.296220+0000","last_active":"2024-04-26T05:06:10.296220+0000","last_peered":"2024-04-26T05:06:10.296220+0000","last_clean":"2024-04-26T05:06:10.296220+0000","last_became_active":"2024-04-26T05:06:10.294923+0000","last_became_peered":"2024-04-26T05:06:10.294923+0000","last_unstale":"2024-04-26T05:06:10.296220+0000","last_undegraded":"2024-04-26T05:06:10.296220+0000","last_fullsized":"2024-04-26T05:06:10.296220+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:26:23.919695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.293058+0000","last_change":"2024-04-26T05:06:10.292656+0000","last_active":"2024-04-26T05:06:10.293058+0000","last_peered":"2024-04-26T05:06:10.293058+0000","last_clean":"2024-04-26T05:06:10.293058+0000","last_became_active":"2024-04-26T05:06:10.291777+0000","last_became_peered":"2024-04-26T05:06:10.291777+0000","last_unstale":"2024-04-26T05:06:10.293058+0000","last_undegraded":"2024-04-26T05:06:10.293058+0000","last_fullsized":"2024-04-26T05:06:10.293058+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:58:24.110953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.279918+0000","last_change":"2024-04-26T05:05:01.299539+0000","last_active":"2024-04-26T05:06:09.279918+0000","last_peered":"2024-04-26T05:06:09.279918+0000","last_clean":"2024-04-26T05:06:09.279918+0000","last_became_active":"2024-04-26T05:05:01.298988+0000","last_became_peered":"2024-04-26T05:05:01.298988+0000","last_unstale":"2024-04-26T05:06:09.279918+0000","last_undegraded":"2024-04-26T05:06:09.279918+0000","last_fullsized":"2024-04-26T05:06:09.279918+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:14:32.254680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.296508+0000","last_change":"2024-04-26T05:06:10.296508+0000","last_active":"2024-04-26T05:06:10.296508+0000","last_peered":"2024-04-26T05:06:10.296508+0000","last_clean":"2024-04-26T05:06:10.296508+0000","last_became_active":"2024-04-26T05:06:10.295018+0000","last_became_peered":"2024-04-26T05:06:10.295018+0000","last_unstale":"2024-04-26T05:06:10.296508+0000","last_undegraded":"2024-04-26T05:06:10.296508+0000","last_fullsized":"2024-04-26T05:06:10.296508+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:19:18.698335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.293577+0000","last_change":"2024-04-26T05:06:10.293577+0000","last_active":"2024-04-26T05:06:10.293577+0000","last_peered":"2024-04-26T05:06:10.293577+0000","last_clean":"2024-04-26T05:06:10.293577+0000","last_became_active":"2024-04-26T05:06:10.293212+0000","last_became_peered":"2024-04-26T05:06:10.293212+0000","last_unstale":"2024-04-26T05:06:10.293577+0000","last_undegraded":"2024-04-26T05:06:10.293577+0000","last_fullsized":"2024-04-26T05:06:10.293577+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:04:10.927936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.288260+0000","last_change":"2024-04-26T05:06:10.288260+0000","last_active":"2024-04-26T05:06:10.288260+0000","last_peered":"2024-04-26T05:06:10.288260+0000","last_clean":"2024-04-26T05:06:10.288260+0000","last_became_active":"2024-04-26T05:06:10.287923+0000","last_became_peered":"2024-04-26T05:06:10.287923+0000","last_unstale":"2024-04-26T05:06:10.288260+0000","last_undegraded":"2024-04-26T05:06:10.288260+0000","last_fullsized":"2024-04-26T05:06:10.288260+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:43:11.502894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2024-04-26T05:06:10.294619+0000","last_change":"2024-04-26T05:06:10.294619+0000","last_active":"2024-04-26T05:06:10.294619+0000","last_peered":"2024-04-26T05:06:10.294619+0000","last_clean":"2024-04-26T05:06:10.294619+0000","last_became_active":"2024-04-26T05:06:10.294215+0000","last_became_peered":"2024-04-26T05:06:10.294215+0000","last_unstale":"2024-04-26T05:06:10.294619+0000","last_undegraded":"2024-04-26T05:06:10.294619+0000","last_fullsized":"2024-04-26T05:06:10.294619+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:08.379513+0000","last_clean_scrub_stamp":"2024-04-26T04:56:08.379513+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:48:28.705784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-04-26T05:06:09.275170+0000","last_change":"2024-04-26T05:05:35.495708+0000","last_active":"2024-04-26T05:06:09.275170+0000","last_peered":"2024-04-26T05:06:09.275170+0000","last_clean":"2024-04-26T05:06:09.275170+0000","last_became_active":"2024-04-26T05:05:35.495401+0000","last_became_peered":"2024-04-26T05:05:35.495401+0000","last_unstale":"2024-04-26T05:06:09.275170+0000","last_undegraded":"2024-04-26T05:06:09.275170+0000","last_fullsized":"2024-04-26T05:06:09.275170+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:56:07.374199+0000","last_clean_scrub_stamp":"2024-04-26T04:56:07.374199+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:04:36.178002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18628,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":11984,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5319,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201437,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307340,"kb_used_data":5888,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93437812,"statfs":{"total":95995035648,"available":95680319488,"internally_reserved":0,"allocated":6029312,"data_stored":5423320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463075,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303244,"kb_used_data":5880,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93441908,"statfs":{"total":95995035648,"available":95684513792,"internally_reserved":0,"allocated":6021120,"data_stored":5423201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":100,"seq":429496729628,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32876,"kb_used_data":1184,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93712276,"statfs":{"total":95995035648,"available":95961370624,"internally_reserved":0,"allocated":1212416,"data_stored":633520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7683,"internal_metadata":32432637},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":95,"seq":408021893162,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33472,"kb_used_data":1776,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93711680,"statfs":{"total":95995035648,"available":95960760320,"internally_reserved":0,"allocated":1818624,"data_stored":1224999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8208,"internal_metadata":32432112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":105,"seq":450971566101,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32828,"kb_used_data":5288,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93712324,"statfs":{"total":95995035648,"available":95961419776,"internally_reserved":0,"allocated":5414912,"data_stored":4828320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8512,"internal_metadata":28171968},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":110,"seq":472446402574,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28724,"kb_used_data":1192,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716428,"statfs":{"total":95995035648,"available":95965622272,"internally_reserved":0,"allocated":1220608,"data_stored":634113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6360,"internal_metadata":28174120},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986348,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298568,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446584,"statfs":{"total":95995035648,"available":95689302016,"internally_reserved":0,"allocated":1236992,"data_stored":638636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724711,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":634397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3920,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1399,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6504,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5979,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2888,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3257,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:07:14.040 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T05:07:14.040 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T05:07:14.040 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T05:07:14.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:14.298 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:14.357 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:07:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:07:14] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.43.0" 2024-04-26T05:07:14.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:14 smithi152 ceph-mon[102000]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:07:14.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:14 smithi152 ceph-mon[106538]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:07:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:14 smithi195 ceph-mon[90222]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:07:15.543 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:15 smithi152 ceph-mon[102000]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:15.543 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:15 smithi152 ceph-mon[106538]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:15.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:15 smithi195 ceph-mon[90222]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:16.304 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:16.305 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:16.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:16 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1859421317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:16.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:16 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1859421317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:16.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:16 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1859421317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:17.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:17 smithi152 ceph-mon[102000]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:17.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:17 smithi152 ceph-mon[106538]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:17 smithi195 ceph-mon[90222]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:18.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:18.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:19.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:19 smithi152 ceph-mon[102000]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:19.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:19 smithi152 ceph-mon[106538]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:19 smithi195 ceph-mon[90222]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:20.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:20.331 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:21.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:21 smithi152 ceph-mon[102000]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:21.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:21 smithi152 ceph-mon[106538]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:21 smithi195 ceph-mon[90222]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:22.425 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:22.426 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:23.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:23 smithi195 ceph-mon[90222]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:23 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3154700768' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:23.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:23 smithi152 ceph-mon[102000]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:23 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3154700768' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:23.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:23 smithi152 ceph-mon[106538]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:23 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3154700768' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:24.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:07:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:07:24] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.43.0" 2024-04-26T05:07:25.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:25 smithi195 ceph-mon[90222]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:25.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:25 smithi152 ceph-mon[102000]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:25.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:25 smithi152 ceph-mon[106538]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:26.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:26.329 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:27.670 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:27 smithi152 ceph-mon[102000]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:27.670 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:27 smithi152 ceph-mon[106538]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:27.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:27 smithi195 ceph-mon[90222]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:28.279 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:28.279 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:28.552 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:28 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:28.552 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:28 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/4205753251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:28.552 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:28 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:28.552 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:28 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/4205753251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:28.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:28 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:28.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:28 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/4205753251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:29.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:29 smithi152 ceph-mon[102000]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:29.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:29 smithi152 ceph-mon[106538]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:29.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:29 smithi195 ceph-mon[90222]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:31.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:31 smithi195 ceph-mon[90222]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:31.824 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:31 smithi152 ceph-mon[102000]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:31.824 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:31 smithi152 ceph-mon[106538]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:31.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:32.341 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:33.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:33.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:33.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:07:34.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:33 smithi195 ceph-mon[90222]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:33 smithi195 ceph-mon[90222]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:33 smithi152 ceph-mon[106538]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:33 smithi152 ceph-mon[106538]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:33 smithi152 ceph-mon[102000]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:33 smithi152 ceph-mon[102000]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.381 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:07:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:07:34] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:07:34.448 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:34.448 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:35.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:34 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2142773477' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:35.131 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:34 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2142773477' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:35.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:34 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2142773477' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:36.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:35 smithi195 ceph-mon[90222]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:36.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:35 smithi152 ceph-mon[102000]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:35 smithi152 ceph-mon[106538]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:38.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:38.488 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:38.611 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:38 smithi152 ceph-mon[102000]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:38.611 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:38 smithi152 ceph-mon[106538]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:38.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:38 smithi195 ceph-mon[90222]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:40.417 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:40.417 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:40.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:40 smithi152 ceph-mon[102000]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:40 smithi152 ceph-mon[106538]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:40.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:40 smithi195 ceph-mon[90222]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:41.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:41 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/58519588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:41.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:41 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/58519588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:41.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:41 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/58519588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:42.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:42 smithi152 ceph-mon[102000]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:42.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:42 smithi152 ceph-mon[106538]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:42.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:42 smithi195 ceph-mon[90222]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:44.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:44.328 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:44.369 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:07:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:07:44] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:07:44.622 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:44 smithi152 ceph-mon[102000]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:44.622 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:44 smithi152 ceph-mon[106538]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:44.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:44 smithi195 ceph-mon[90222]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:46.313 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:46.314 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:46.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:46 smithi152 ceph-mon[106538]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:46.624 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:46 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1424992904' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:46.624 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:46 smithi152 ceph-mon[102000]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:46.625 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:46 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1424992904' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:46.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:46 smithi195 ceph-mon[90222]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:46.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:46 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1424992904' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:48 smithi152 ceph-mon[102000]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:48 smithi152 ceph-mon[106538]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:48.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:48 smithi195 ceph-mon[90222]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:48.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:49.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:50.191 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:50.567 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:50 smithi152 ceph-mon[102000]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:50.567 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:50 smithi152 ceph-mon[106538]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:50.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:50 smithi195 ceph-mon[90222]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:52.015 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:52.016 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:52.604 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:52 smithi152 ceph-mon[102000]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:52.605 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:52 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3054494208' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:52.605 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:52 smithi152 ceph-mon[106538]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:52.605 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:52 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3054494208' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:52.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:52 smithi195 ceph-mon[90222]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:52.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:52 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3054494208' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:54.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:54 smithi152 ceph-mon[102000]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:54.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:54 smithi152 ceph-mon[106538]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:54.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:07:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:07:54] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:07:54.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:54 smithi195 ceph-mon[90222]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:55.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:55.896 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:56 smithi152 ceph-mon[102000]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:56.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:56 smithi152 ceph-mon[106538]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:56.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:56 smithi195 ceph-mon[90222]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:57.828 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:07:57.828 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:58.474 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:58 smithi152 ceph-mon[106538]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:58.474 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:07:58 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/956073947' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:58.474 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:58 smithi152 ceph-mon[102000]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:58.474 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:07:58 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/956073947' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:58.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:58 smithi195 ceph-mon[90222]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:58.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:07:58 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/956073947' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:00.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:00 smithi195 ceph-mon[90222]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:00.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:00 smithi152 ceph-mon[102000]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:00.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:00 smithi152 ceph-mon[106538]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:01.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:01.741 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:02.598 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:02 smithi152 ceph-mon[102000]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:02.599 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:02 smithi152 ceph-mon[106538]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:02.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:02 smithi195 ceph-mon[90222]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:03.478 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:03.478 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:03.705 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:03.705 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:03.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:04.306 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:08:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:08:04] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:08:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:04 smithi152 ceph-mon[102000]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:04.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:04 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1905447826' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:04 smithi152 ceph-mon[106538]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:04.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:04 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1905447826' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:04.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:04 smithi195 ceph-mon[90222]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:04.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:04 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1905447826' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:06 smithi195 ceph-mon[90222]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:06.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:06 smithi152 ceph-mon[102000]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:06.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:06 smithi152 ceph-mon[106538]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:07.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:07.620 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:08 smithi195 ceph-mon[90222]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:08.788 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:08 smithi152 ceph-mon[106538]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:08.788 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:08 smithi152 ceph-mon[102000]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:09.594 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:09.594 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:10 smithi195 ceph-mon[90222]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:10 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/4276651380' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:10.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:10 smithi152 ceph-mon[102000]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:10.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:10 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/4276651380' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:10.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:10 smithi152 ceph-mon[106538]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:10.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:10 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/4276651380' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:12 smithi195 ceph-mon[90222]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:12.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:12 smithi152 ceph-mon[102000]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:12.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:12 smithi152 ceph-mon[106538]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:13.237 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:13.508 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:14.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:08:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:08:14] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:08:14.759 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:14 smithi152 ceph-mon[102000]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:14.759 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:14 smithi152 ceph-mon[106538]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:14 smithi195 ceph-mon[90222]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:15.488 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:15.488 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:16.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:16 smithi195 ceph-mon[90222]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:16.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:16 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1865216505' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:16.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:16 smithi152 ceph-mon[102000]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:16.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:16 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1865216505' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:16.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:16 smithi152 ceph-mon[106538]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:16.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:16 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1865216505' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:17.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:17 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:17.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:17 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:17.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:17 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:18.672 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:18 smithi152 ceph-mon[106538]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:18.672 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:18 smithi152 ceph-mon[102000]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:18 smithi195 ceph-mon[90222]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:19.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:19.442 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:20.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:20 smithi152 ceph-mon[102000]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:20.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:20 smithi152 ceph-mon[106538]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:20.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:20 smithi195 ceph-mon[90222]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:21.445 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:21.445 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:22 smithi195 ceph-mon[90222]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:22.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:22 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3657860865' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:22.882 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:22 smithi152 ceph-mon[102000]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:22.883 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:22 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3657860865' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:22.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:22 smithi152 ceph-mon[106538]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:22.884 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:22 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3657860865' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:24.430 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:08:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:08:24] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:08:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:24 smithi195 ceph-mon[90222]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:24.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:24 smithi152 ceph-mon[102000]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:24.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:24 smithi152 ceph-mon[106538]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:25.389 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:25.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:25 smithi195 ceph-mon[90222]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.815 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:25 smithi152 ceph-mon[102000]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.815 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:25 smithi152 ceph-mon[106538]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:27.497 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:27.497 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:28.093 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:28 smithi152 ceph-mon[102000]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:28.093 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:28 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/450619375' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:28.093 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:28 smithi152 ceph-mon[106538]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:28.093 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:28 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/450619375' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:28.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:28 smithi195 ceph-mon[90222]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:28.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:28 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/450619375' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:30.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:30 smithi195 ceph-mon[90222]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:30.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:30 smithi152 ceph-mon[102000]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:30 smithi152 ceph-mon[106538]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:31.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:31.384 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:32.129 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:32 smithi152 ceph-mon[102000]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:32.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:32 smithi152 ceph-mon[106538]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:32.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:32 smithi195 ceph-mon[90222]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:33.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:33.436 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:33.436 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:33.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:34.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:34 smithi152 ceph-mon[102000]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:34.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:34 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:34.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:34 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1880058553' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:34.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:34 smithi152 ceph-mon[106538]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:34.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:34 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:34.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:34 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1880058553' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:34.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:34 smithi195 ceph-mon[90222]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:34.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:34 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:34.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:34 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1880058553' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:34.611 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:08:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:08:34] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.43.0" 2024-04-26T05:08:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:36 smithi152 ceph-mon[102000]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:36 smithi152 ceph-mon[106538]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:36.483 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:36 smithi195 ceph-mon[90222]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:37.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:37.496 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:38.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:37 smithi195 ceph-mon[90222]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:37 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.371 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[106538]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[102000]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:38.372 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:37 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:39.067 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:39.067 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:39.068 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:38 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:39.068 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:39.068 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:39.068 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:39.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:39.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:39.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:38 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:08:39.557 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:39.557 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:40.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[102000]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[102000]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[102000]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/417966609' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:40.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[106538]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[106538]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[106538]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.131 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:40 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/417966609' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:40.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:40 smithi195 ceph-mon[90222]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:40 smithi195 ceph-mon[90222]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:40 smithi195 ceph-mon[90222]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:40 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/417966609' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:42.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:42 smithi195 ceph-mon[90222]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:42 smithi152 ceph-mon[102000]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:42 smithi152 ceph-mon[106538]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:43.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:43.558 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:44.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:44 smithi195 ceph-mon[90222]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:44.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:44 smithi152 ceph-mon[102000]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:44.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:08:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:08:44] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.43.0" 2024-04-26T05:08:44.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:44 smithi152 ceph-mon[106538]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:45.760 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:46.067 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:46 smithi152 ceph-mon[102000]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:46.067 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:46 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2606479177' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:46.068 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:46 smithi152 ceph-mon[106538]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:46.068 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:46 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2606479177' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:46.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:46 smithi195 ceph-mon[90222]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:46.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:46 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2606479177' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:48.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:48 smithi195 ceph-mon[90222]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:48.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:48.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:48 smithi152 ceph-mon[102000]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:48.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:48.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:48 smithi152 ceph-mon[106538]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:48.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:49.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:49.693 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:50.079 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:50 smithi152 ceph-mon[102000]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:50.079 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:50 smithi152 ceph-mon[106538]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:50.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:50 smithi195 ceph-mon[90222]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:51.722 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:51.722 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:52.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:52 smithi195 ceph-mon[90222]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:52.281 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:52 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1701213778' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:52.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:52 smithi152 ceph-mon[102000]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:52.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:52 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1701213778' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:52.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:52 smithi152 ceph-mon[106538]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:52.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:52 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1701213778' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:54.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:54 smithi195 ceph-mon[90222]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:54.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:54 smithi152 ceph-mon[102000]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:54.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:08:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:08:54] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2024-04-26T05:08:54.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:54 smithi152 ceph-mon[106538]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:55.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:55.691 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:56.126 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:56 smithi152 ceph-mon[102000]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:56.127 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:56 smithi152 ceph-mon[106538]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:56.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:56 smithi195 ceph-mon[90222]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:57.541 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:08:57.541 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:58.203 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:58 smithi152 ceph-mon[102000]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:58.204 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:08:58 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2282538813' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:58.204 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:58 smithi152 ceph-mon[106538]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:58.204 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:08:58 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2282538813' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:58.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:58 smithi195 ceph-mon[90222]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:58.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:08:58 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2282538813' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:00.280 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:00 smithi195 ceph-mon[90222]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:00 smithi152 ceph-mon[102000]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:00 smithi152 ceph-mon[106538]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:01.231 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:01.500 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:02.310 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:02 smithi152 ceph-mon[102000]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:02.310 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:02 smithi152 ceph-mon[106538]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:02.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:02 smithi195 ceph-mon[90222]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:03.198 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:03.199 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:03.425 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:03.425 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:03.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:04.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:09:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:09:04] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:09:04.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:04 smithi152 ceph-mon[102000]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:04.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:04 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/4264306351' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:04.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:04 smithi152 ceph-mon[106538]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:04.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:04 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/4264306351' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:04 smithi195 ceph-mon[90222]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:04 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/4264306351' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:06.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:06 smithi152 ceph-mon[106538]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:06.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:06 smithi152 ceph-mon[102000]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:06.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:06 smithi195 ceph-mon[90222]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:07.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:07.359 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:08.248 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:08 smithi152 ceph-mon[102000]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:08.248 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:08 smithi152 ceph-mon[106538]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:08.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:08 smithi195 ceph-mon[90222]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:09.379 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:09.379 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:10 smithi152 ceph-mon[102000]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:10.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:10 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2926798320' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:10.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:10 smithi152 ceph-mon[106538]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:10.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:10 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2926798320' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:10.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:10 smithi195 ceph-mon[90222]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:10.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:10 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2926798320' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:12 smithi152 ceph-mon[102000]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:12.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:12 smithi152 ceph-mon[106538]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:12.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:12 smithi195 ceph-mon[90222]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:12.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:13.241 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:14.333 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:14 smithi195 ceph-mon[90222]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:14 smithi152 ceph-mon[102000]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:09:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:09:14] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:09:14.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:14 smithi152 ceph-mon[106538]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:15.166 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:15.167 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:16.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:16 smithi152 ceph-mon[102000]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:16.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:16 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1793455494' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:16.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:16 smithi152 ceph-mon[106538]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:16.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:16 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1793455494' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:16 smithi195 ceph-mon[90222]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:16.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:16 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1793455494' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:18.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:18 smithi152 ceph-mon[102000]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:18.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:18.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:18 smithi152 ceph-mon[106538]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:18.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:18 smithi195 ceph-mon[90222]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:18.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:18.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:19.078 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:20.516 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:20 smithi152 ceph-mon[102000]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:20.516 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:20 smithi152 ceph-mon[106538]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:20.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:20 smithi195 ceph-mon[90222]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:21.013 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:21.013 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:21.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:21 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/693503178' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:21.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:21 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/693503178' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:21.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:21 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/693503178' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:22.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:22 smithi195 ceph-mon[90222]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:22.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:22 smithi152 ceph-mon[102000]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:22.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:22 smithi152 ceph-mon[106538]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:24.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:24 smithi152 ceph-mon[102000]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:24.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:24 smithi152 ceph-mon[106538]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:24.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:09:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:09:24] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.43.0" 2024-04-26T05:09:24.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:24.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:24 smithi195 ceph-mon[90222]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:24.954 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:26.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:26 smithi152 ceph-mon[102000]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:26.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:26 smithi152 ceph-mon[106538]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:26.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:26 smithi195 ceph-mon[90222]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:26.824 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:26.824 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:27.494 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:27 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/942587520' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:27.495 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:27 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/942587520' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:27.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:27 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/942587520' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:28.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:28 smithi195 ceph-mon[90222]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:28.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:28 smithi152 ceph-mon[102000]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:28.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:28 smithi152 ceph-mon[106538]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:30.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:30.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:30 smithi195 ceph-mon[90222]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:30.788 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:30.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:30 smithi152 ceph-mon[102000]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:30.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:30 smithi152 ceph-mon[106538]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:32.496 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:32.497 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:32.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:32 smithi195 ceph-mon[90222]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:32.789 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:32 smithi152 ceph-mon[102000]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:32.790 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:32 smithi152 ceph-mon[106538]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:33.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:33.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:33 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1438149595' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:33.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:33.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:33 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1438149595' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:33.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:33.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:33 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1438149595' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:34.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:34 smithi152 ceph-mon[102000]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:34.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:34 smithi152 ceph-mon[106538]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:34.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:09:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:09:34] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:09:34.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:34 smithi195 ceph-mon[90222]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:36.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:36.399 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:36.432 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:36 smithi152 ceph-mon[102000]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:36.432 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:36 smithi152 ceph-mon[106538]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:36.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:36 smithi195 ceph-mon[90222]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:38.395 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:38.704 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:38 smithi152 ceph-mon[102000]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:38.704 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:38 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3014159200' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:38.704 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:38 smithi152 ceph-mon[106538]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:38.704 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:38 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3014159200' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:38.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:38 smithi195 ceph-mon[90222]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:38.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:38 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3014159200' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:39.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:39 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:09:39.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:39 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:09:39.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:39 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:09:40.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:40 smithi152 ceph-mon[102000]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:40 smithi152 ceph-mon[106538]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:40.738 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:40 smithi195 ceph-mon[90222]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:42.056 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:42.320 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:42.684 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:42 smithi152 ceph-mon[106538]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:42.684 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:42 smithi152 ceph-mon[102000]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:42.705 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:42 smithi195 ceph-mon[90222]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.347 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:09:44] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:09:44.347 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[102000]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.347 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.347 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.347 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.348 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.348 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[106538]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.348 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.348 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.348 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.348 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:44 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:44 smithi195 ceph-mon[90222]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:44 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:44.951 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:44.951 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:45.267 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:09:45.267 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:09:45.267 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:45.267 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3098043905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:45.268 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:09:45.268 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:09:45.268 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:45.268 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:45 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3098043905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:45 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:09:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:45 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:09:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:45 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:09:45.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:45 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3098043905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:46 smithi152 ceph-mon[106538]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:46 smithi152 ceph-mon[106538]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:46 smithi152 ceph-mon[102000]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:46 smithi152 ceph-mon[102000]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:46 smithi195 ceph-mon[90222]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:46.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:46 smithi195 ceph-mon[90222]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:48.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:48 smithi152 ceph-mon[106538]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:48.381 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:48.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:48 smithi152 ceph-mon[102000]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:48.382 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:48 smithi195 ceph-mon[90222]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:48.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:48.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:48.976 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:50.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:50 smithi152 ceph-mon[102000]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:50.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:50 smithi152 ceph-mon[106538]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:50 smithi195 ceph-mon[90222]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:51.004 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:51.004 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:51.355 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:51 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/957226137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:51.356 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:51 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/957226137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:51.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:51 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/957226137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:52.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:52 smithi152 ceph-mon[106538]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:52.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:52 smithi152 ceph-mon[102000]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:52.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:52 smithi195 ceph-mon[90222]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:54.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:54 smithi152 ceph-mon[102000]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:54.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:54 smithi152 ceph-mon[106538]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:54.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:09:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:09:54] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:09:54.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:54 smithi195 ceph-mon[90222]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:54.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:55.041 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:56.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:56 smithi152 ceph-mon[102000]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:56.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:56 smithi152 ceph-mon[106538]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:56.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:56 smithi195 ceph-mon[90222]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:57.077 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:09:57.077 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:57.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:57 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2018541725' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:57.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:57 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2018541725' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:57.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:57 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2018541725' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:58.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:09:58 smithi152 ceph-mon[102000]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:58.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:09:58 smithi152 ceph-mon[106538]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:58.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:09:58 smithi195 ceph-mon[90222]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:00 smithi152 ceph-mon[102000]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:00.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:00 smithi152 ceph-mon[102000]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:10:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:00 smithi152 ceph-mon[106538]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:00.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:00 smithi152 ceph-mon[106538]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:10:00.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:00 smithi195 ceph-mon[90222]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:00.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:00 smithi195 ceph-mon[90222]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:10:00.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:00.940 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:02.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:02 smithi152 ceph-mon[102000]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:02.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:02 smithi152 ceph-mon[106538]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:02.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:02 smithi195 ceph-mon[90222]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:02.914 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:02.914 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:03.224 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:03.225 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:03 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2076362876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:03.225 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:03.225 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:03 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2076362876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:03.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:03.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:03 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2076362876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:04.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:04 smithi152 ceph-mon[106538]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:04.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:04 smithi152 ceph-mon[102000]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:04.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:10:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:10:04] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:10:04.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:04 smithi195 ceph-mon[90222]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:06.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:06 smithi152 ceph-mon[102000]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:06.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:06 smithi152 ceph-mon[106538]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:06.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:06 smithi195 ceph-mon[90222]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:06.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:06.894 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:08.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:08 smithi152 ceph-mon[102000]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:08.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:08 smithi152 ceph-mon[106538]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:08.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:08 smithi195 ceph-mon[90222]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:08.872 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:08.872 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:09.130 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:09 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/4135780478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:09.130 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:09 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/4135780478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:09.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:09 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/4135780478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:10.379 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:10 smithi152 ceph-mon[102000]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:10.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:10 smithi152 ceph-mon[106538]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:10.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:10 smithi195 ceph-mon[90222]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:12.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:12 smithi152 ceph-mon[102000]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:12.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:12 smithi152 ceph-mon[106538]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:12.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:12.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:12 smithi195 ceph-mon[90222]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:12.704 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:14.311 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:14 smithi152 ceph-mon[102000]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:14.312 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:10:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:10:14] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:10:14.312 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:14 smithi152 ceph-mon[106538]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:14.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:14 smithi195 ceph-mon[90222]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:14.567 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:14.567 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:15.223 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:15 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2366980976' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:15.223 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:15 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2366980976' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:15.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:15 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2366980976' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:16.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:16 smithi152 ceph-mon[106538]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:16.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:16 smithi152 ceph-mon[102000]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:16.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:16 smithi195 ceph-mon[90222]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:18.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:18.379 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:18 smithi152 ceph-mon[106538]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:18.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:18.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:18 smithi152 ceph-mon[102000]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:18.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:18.516 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:18.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:18 smithi195 ceph-mon[90222]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:18.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:20.227 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:20 smithi152 ceph-mon[102000]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:20.227 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:20 smithi152 ceph-mon[106538]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:20.470 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:20.470 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:20.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:20 smithi195 ceph-mon[90222]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:21.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:21 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1551310490' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:21.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:21 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1551310490' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:21.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:21 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1551310490' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:22.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:22 smithi195 ceph-mon[90222]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:22.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:22 smithi152 ceph-mon[102000]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:22.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:22 smithi152 ceph-mon[106538]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:24.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:24.330 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:24.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:24 smithi152 ceph-mon[102000]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:24.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:24 smithi152 ceph-mon[106538]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:24.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:10:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:10:24] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:10:24.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:24 smithi195 ceph-mon[90222]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:26.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:26 smithi152 ceph-mon[102000]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:26.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:26 smithi152 ceph-mon[106538]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:26.425 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:26.425 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:26.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:26 smithi195 ceph-mon[90222]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:27.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:27 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2845533975' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:27.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:27 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2845533975' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:27.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:27 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2845533975' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:28.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:28 smithi195 ceph-mon[90222]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:28.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:28 smithi152 ceph-mon[102000]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:28.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:28 smithi152 ceph-mon[106538]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:30.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:30.339 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:30.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:30 smithi152 ceph-mon[102000]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:30.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:30 smithi152 ceph-mon[106538]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:30.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:30 smithi195 ceph-mon[90222]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:32.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:32 smithi152 ceph-mon[102000]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:32.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:32 smithi152 ceph-mon[106538]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:32.410 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:32.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:32 smithi195 ceph-mon[90222]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:33.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:33 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1171479513' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:33.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:33 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:33 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1171479513' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:33.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:33 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:33.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:33 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1171479513' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:33.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:33 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:34.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:34 smithi195 ceph-mon[90222]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:34.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:34 smithi152 ceph-mon[102000]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:34.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:34 smithi152 ceph-mon[106538]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:34.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:10:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:10:34] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:10:36.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:36.363 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:36.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:36 smithi152 ceph-mon[102000]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:36.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:36 smithi152 ceph-mon[106538]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:36.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:36 smithi195 ceph-mon[90222]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:38.456 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:38.457 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:38.457 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:38 smithi152 ceph-mon[102000]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:38.457 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:38 smithi152 ceph-mon[106538]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:38.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:38 smithi195 ceph-mon[90222]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:39.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:39 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/448266321' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:39.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:39 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/448266321' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:39.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:39 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/448266321' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:40.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:40 smithi195 ceph-mon[90222]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:40.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:40 smithi152 ceph-mon[102000]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:40.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:40 smithi152 ceph-mon[106538]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:42.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:42.364 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:42.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:42 smithi152 ceph-mon[102000]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:42.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:42 smithi152 ceph-mon[106538]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:42.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:42 smithi195 ceph-mon[90222]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:44.256 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:44.256 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:44.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:44 smithi195 ceph-mon[90222]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:44.558 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:44 smithi152 ceph-mon[102000]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:44.559 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:10:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:10:44] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:10:44.559 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:44 smithi152 ceph-mon[106538]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:45 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/638294624' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:45.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:45 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:10:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:45 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/638294624' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:45.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:45 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:10:45.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:45 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/638294624' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:45.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:45 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:10:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[106538]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.380 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[102000]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:46.380 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.381 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:46 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:46 smithi195 ceph-mon[90222]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:46.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:46.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:46 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:47.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:48.406 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:48 smithi195 ceph-mon[90222]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:48.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:48 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:48.629 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:48 smithi152 ceph-mon[102000]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:48.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:48 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:48 smithi152 ceph-mon[106538]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:48.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:48 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:50.436 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[106538]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:50.436 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.436 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.436 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.437 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.437 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[102000]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:50.437 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.437 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.437 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.437 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:50 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:50 smithi195 ceph-mon[90222]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:50 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:50.928 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:50.928 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:51.283 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:10:51.283 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:10:51.283 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[102000]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.283 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[102000]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.283 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:51.284 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2900865698' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:51.284 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:10:51.284 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:10:51.284 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[106538]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.284 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[106538]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.284 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:51.284 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:51 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2900865698' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:51.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:51 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:10:51.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:51 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:10:51.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:51 smithi195 ceph-mon[90222]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:51 smithi195 ceph-mon[90222]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:51 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:10:51.532 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:51 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2900865698' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:53.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:53 smithi195 ceph-mon[90222]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:53 smithi152 ceph-mon[102000]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:53.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:53 smithi152 ceph-mon[106538]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:54.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:54.601 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:10:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:10:54] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.43.0" 2024-04-26T05:10:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:55.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:55 smithi152 ceph-mon[102000]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:55.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:55 smithi152 ceph-mon[106538]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:55.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:55 smithi195 ceph-mon[90222]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:56.945 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:10:56.945 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:57.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:57 smithi152 ceph-mon[102000]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:57.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:57 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3520722567' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:57 smithi152 ceph-mon[106538]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:57.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:57 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3520722567' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:57.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:57 smithi195 ceph-mon[90222]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:57.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:57 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3520722567' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:59.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:10:59 smithi195 ceph-mon[90222]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:59.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:10:59 smithi152 ceph-mon[102000]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:59.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:10:59 smithi152 ceph-mon[106538]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:00.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:01.000 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:01.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:01 smithi152 ceph-mon[102000]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:01.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:01 smithi152 ceph-mon[106538]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:01.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:01 smithi195 ceph-mon[90222]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:03.053 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:03.053 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:03.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:03 smithi195 ceph-mon[90222]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:03 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:03.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:03 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/421259108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:03.865 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:03 smithi152 ceph-mon[106538]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:03.865 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:03 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:03.865 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:03 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/421259108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:03.866 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:03 smithi152 ceph-mon[102000]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:03.866 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:03 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:03.866 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:03 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/421259108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:04.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:11:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:11:04] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.43.0" 2024-04-26T05:11:05.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:05 smithi195 ceph-mon[90222]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:05.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:05 smithi152 ceph-mon[102000]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:05.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:05 smithi152 ceph-mon[106538]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:06.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:07.156 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:07.547 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:07 smithi152 ceph-mon[102000]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:07.547 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:07 smithi152 ceph-mon[106538]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:07.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:07 smithi195 ceph-mon[90222]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:09.201 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:09.201 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:09.523 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:09 smithi152 ceph-mon[102000]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:09.523 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:09 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3947759924' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:09.523 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:09 smithi152 ceph-mon[106538]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:09.523 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:09 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3947759924' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:09.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:09 smithi195 ceph-mon[90222]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:09.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:09 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3947759924' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:11.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:11 smithi195 ceph-mon[90222]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:11.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:11 smithi152 ceph-mon[102000]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:11.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:11 smithi152 ceph-mon[106538]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:12.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:13.208 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:13.618 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:13 smithi152 ceph-mon[102000]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:13.618 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:13 smithi152 ceph-mon[106538]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:13.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:13 smithi195 ceph-mon[90222]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:14.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:11:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:11:14] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.43.0" 2024-04-26T05:11:15.209 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:15.209 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:15.521 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:15 smithi152 ceph-mon[102000]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:15.521 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:15 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3764742952' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:15.521 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:15 smithi152 ceph-mon[106538]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:15.522 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:15 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3764742952' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:15.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:15 smithi195 ceph-mon[90222]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:15.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:15 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3764742952' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:17.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:17 smithi195 ceph-mon[90222]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:17.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:17 smithi152 ceph-mon[102000]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:17.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:17 smithi152 ceph-mon[106538]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:18.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:18 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:18.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:18.873 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:18 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:18.873 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:18 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:19.108 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:19.574 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:19 smithi152 ceph-mon[102000]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:19.575 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:19 smithi152 ceph-mon[106538]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:19.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:19 smithi195 ceph-mon[90222]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:21.123 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:21.123 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:21 smithi195 ceph-mon[90222]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:21.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:21 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/2858398798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:21.804 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:21 smithi152 ceph-mon[102000]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:21.804 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:21 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/2858398798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:21.804 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:21 smithi152 ceph-mon[106538]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:21.804 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:21 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/2858398798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:23.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:23 smithi195 ceph-mon[90222]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:23.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:23 smithi152 ceph-mon[102000]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:23.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:23 smithi152 ceph-mon[106538]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:24.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:11:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:11:24] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.43.0" 2024-04-26T05:11:24.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:25.095 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:25.524 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:25 smithi152 ceph-mon[102000]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:25.524 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:25 smithi152 ceph-mon[106538]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:25.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:25 smithi195 ceph-mon[90222]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:27.147 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:27.147 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:27.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:27 smithi195 ceph-mon[90222]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:27.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:27 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/174182897' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:27.794 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:27 smithi152 ceph-mon[106538]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:27.794 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:27 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/174182897' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:27.794 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:27 smithi152 ceph-mon[102000]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:27.794 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:27 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/174182897' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:29.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:29 smithi195 ceph-mon[90222]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:29.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:29 smithi152 ceph-mon[102000]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:29.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:29 smithi152 ceph-mon[106538]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:30.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:31.084 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:31.488 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:31 smithi152 ceph-mon[102000]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:31.488 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:31 smithi152 ceph-mon[106538]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:31.782 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:31 smithi195 ceph-mon[90222]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:32.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:32 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:32.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:32 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:32.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:32 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:33.145 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:33.145 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:33.700 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:33 smithi152 ceph-mon[102000]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:33.701 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:33 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/760901775' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:33.701 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:33 smithi152 ceph-mon[106538]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:33.701 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:33 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/760901775' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:33.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:33 smithi195 ceph-mon[90222]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:33.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:33 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/760901775' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:34.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:11:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:11:34] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:11:35.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:35 smithi195 ceph-mon[90222]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:35.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:35 smithi152 ceph-mon[102000]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:35.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:35 smithi152 ceph-mon[106538]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:36.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:36.993 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:37.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:37 smithi152 ceph-mon[102000]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:37.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:37 smithi152 ceph-mon[106538]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:37.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:37 smithi195 ceph-mon[90222]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:38.954 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:38.955 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:39.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:39 smithi152 ceph-mon[102000]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:39.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:39 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/771139304' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:39.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:39 smithi152 ceph-mon[106538]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:39.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:39 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/771139304' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:39.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:39 smithi195 ceph-mon[90222]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:39.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:39 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/771139304' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:41.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:41 smithi195 ceph-mon[90222]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:41.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:41 smithi152 ceph-mon[102000]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:41.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:41 smithi152 ceph-mon[106538]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:42.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:42.955 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:43.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:43 smithi152 ceph-mon[102000]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:43.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:43 smithi152 ceph-mon[106538]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:43.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:43 smithi195 ceph-mon[90222]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:44.433 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:11:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:11:44] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-04-26T05:11:44.899 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:44.899 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:45.552 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:45 smithi152 ceph-mon[102000]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:45.552 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:45 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/94756146' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:45.552 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:45 smithi152 ceph-mon[106538]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:45.552 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:45 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/94756146' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:45.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:45 smithi195 ceph-mon[90222]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:45.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:45 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/94756146' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:46.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:46 smithi195 ceph-mon[90222]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:46.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:46 smithi152 ceph-mon[102000]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:46.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:46 smithi152 ceph-mon[106538]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:47.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:47 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:47.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:47 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:47.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:47 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:11:48.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:48.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:48 smithi195 ceph-mon[90222]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:48.843 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:48.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:48 smithi152 ceph-mon[102000]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:48.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:48 smithi152 ceph-mon[106538]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:50.782 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:50.783 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:51.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:51 smithi195 ceph-mon[90222]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:51 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:11:51.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:51 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/3828115363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:51.788 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:51 smithi152 ceph-mon[102000]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:51.788 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:51 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:11:51.788 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:51 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/3828115363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:51.788 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:51 smithi152 ceph-mon[106538]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:51.788 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:51 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:11:51.788 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:51 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/3828115363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:53.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:53 smithi195 ceph-mon[90222]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:53.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:53 smithi152 ceph-mon[102000]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:53.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:53 smithi152 ceph-mon[106538]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:54.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:11:54.477 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:11:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:11:54] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-26T05:11:54.729 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:11:55.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:11:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:11:55.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:55 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:11:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:11:55.881 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:55 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' 2024-04-26T05:11:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:56 smithi152 ceph-mon[102000]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:56.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:56 smithi152 ceph-mon[102000]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:56.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:56 smithi152 ceph-mon[106538]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:56.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:56 smithi152 ceph-mon[106538]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:56.764 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:11:56.764 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:11:56.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:56 smithi195 ceph-mon[90222]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:56.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:56 smithi195 ceph-mon[90222]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:57.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:57 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1807901442' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:57.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:57 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1807901442' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:57.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:57 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1807901442' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:11:58.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:11:58 smithi195 ceph-mon[90222]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:58.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:11:58 smithi152 ceph-mon[102000]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:11:58.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:11:58 smithi152 ceph-mon[106538]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:00.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:12:00.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:00 smithi195 ceph-mon[90222]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:00.831 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:12:00.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:00 smithi152 ceph-mon[102000]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:00.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:00 smithi152 ceph-mon[106538]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:02.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:02 smithi152 ceph-mon[102000]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:02.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:02 smithi152 ceph-mon[102000]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:12:02.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:02 smithi152 ceph-mon[106538]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:02.631 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:02 smithi152 ceph-mon[106538]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:12:02.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:02 smithi195 ceph-mon[90222]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:02.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:02 smithi195 ceph-mon[90222]: from='mgr.25898 172.21.15.152:0/680899650' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:12:02.827 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:12:02.828 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:12:03.589 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:03 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1443975420' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:03.589 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:03 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1443975420' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:03.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:03 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1443975420' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:04.439 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:12:04] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.43.0" 2024-04-26T05:12:04.781 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:04 smithi195 ceph-mon[90222]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:04.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:04 smithi152 ceph-mon[102000]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:04.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:04 smithi152 ceph-mon[106538]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:06.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:12:06.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:06 smithi195 ceph-mon[90222]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:06.879 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:06 smithi152 ceph-mon[102000]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:06.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:06 smithi152 ceph-mon[106538]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:06.882 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:12:08.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:08 smithi195 ceph-mon[90222]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:08.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:08 smithi152 ceph-mon[102000]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:08.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:08 smithi152 ceph-mon[106538]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:08.924 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:12:08.925 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:12:09.606 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:09 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/1468297383' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:09.606 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:09 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/1468297383' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:09.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:09 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/1468297383' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:10.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:10 smithi195 ceph-mon[90222]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:10.837 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:10 smithi152 ceph-mon[102000]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:10.838 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:10 smithi152 ceph-mon[106538]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:12.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 164326a4-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:12:12.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:12 smithi195 ceph-mon[90222]: pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:12.880 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:12 smithi152 ceph-mon[102000]: pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:12.880 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:12 smithi152 ceph-mon[106538]: pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:12.899 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:12:14.450 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: ::ffff:172.21.15.195 - - [26/Apr/2024:05:12:14] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.43.0" 2024-04-26T05:12:14.710 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:14 smithi152 ceph-mon[102000]: pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:14.710 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:14 smithi152 ceph-mon[106538]: pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:14.780 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:14 smithi195 ceph-mon[90222]: pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:12:14.952 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-26T05:12:14.952 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:12:15.581 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:15 smithi152 ceph-mon[102000]: from='client.? 172.21.15.152:0/504803632' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:15.581 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:15 smithi152 ceph-mon[106538]: from='client.? 172.21.15.152:0/504803632' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:15.606 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:12:15.847 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=1ab6213493e94b6fbbfbc49484b5f6d5 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:12:15.849 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-26T05:12:15.860 INFO:tasks.cephadm:Teardown begin 2024-04-26T05:12:15.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:12:15.864 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:15 smithi195 ceph-mon[90222]: from='client.? 172.21.15.152:0/504803632' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:12:15.893 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:12:15.919 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-26T05:12:15.920 DEBUG:teuthology.orchestra.run.smithi152:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:12:15.950 DEBUG:teuthology.orchestra.run.smithi195:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:12:15.974 INFO:tasks.cephadm:Stopping all daemons... 2024-04-26T05:12:15.975 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-26T05:12:15.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a 2024-04-26T05:12:16.288 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:16 smithi152 systemd[1]: Stopping Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:16.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a[101977]: 2024-04-26T05:12:16.287+0000 7fcc8bbb8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:16.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a[101977]: 2024-04-26T05:12:16.287+0000 7fcc8bbb8700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-26T05:12:16.630 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:16 smithi152 podman[155410]: 2024-04-26 05:12:16.430331949 +0000 UTC m=+0.248717610 container died 1b4ec8125acddd75e92928f987b7ec1313aadc1647cab0ec97cbeeae5fb9feeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:12:16.630 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:12:16] ENGINE Bus STOPPING 2024-04-26T05:12:16.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:12:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T05:12:16.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:12:16] ENGINE Bus STOPPED 2024-04-26T05:12:16.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:12:16] ENGINE Bus STARTING 2024-04-26T05:12:16.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:12:16] ENGINE Serving on http://:::9283 2024-04-26T05:12:16.631 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:16 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: [26/Apr/2024:05:12:16] ENGINE Bus STARTED 2024-04-26T05:12:17.095 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:16 smithi152 podman[155410]: 2024-04-26 05:12:16.926975773 +0000 UTC m=+0.745361479 container cleanup 1b4ec8125acddd75e92928f987b7ec1313aadc1647cab0ec97cbeeae5fb9feeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T05:12:17.095 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:16 smithi152 bash[155410]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a 2024-04-26T05:12:17.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service' 2024-04-26T05:12:17.364 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:17 smithi152 podman[155441]: 2024-04-26 05:12:17.095328837 +0000 UTC m=+0.661381204 container remove 1b4ec8125acddd75e92928f987b7ec1313aadc1647cab0ec97cbeeae5fb9feeb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, RELEASE=squid-b22e2eb) 2024-04-26T05:12:17.365 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:17 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T05:12:17.365 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:17 smithi152 systemd[1]: Stopped Ceph mon.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:12:17.365 INFO:journalctl@ceph.mon.a.smithi152.stdout:Apr 26 05:12:17 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.a.service: Consumed 9.677s CPU time. 2024-04-26T05:12:17.388 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:17.388 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-26T05:12:17.389 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-26T05:12:17.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.c 2024-04-26T05:12:17.630 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:17 smithi152 systemd[1]: Stopping Ceph mon.c for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:18.034 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:17 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c[106515]: 2024-04-26T05:12:17.717+0000 7fb5d5d80700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:18.034 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:17 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c[106515]: 2024-04-26T05:12:17.717+0000 7fb5d5d80700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:12:18.034 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:17 smithi152 podman[155554]: 2024-04-26 05:12:17.739418074 +0000 UTC m=+0.133097918 container died 0bfb660a4a44fa6e6b8f02247942cb4c6942e91ca8289bf4ed5d03382c7df4d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:12:18.376 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:18 smithi152 podman[155554]: 2024-04-26 05:12:18.132404828 +0000 UTC m=+0.526084662 container cleanup 0bfb660a4a44fa6e6b8f02247942cb4c6942e91ca8289bf4ed5d03382c7df4d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T05:12:18.376 INFO:journalctl@ceph.mon.c.smithi152.stdout:Apr 26 05:12:18 smithi152 bash[155554]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-c 2024-04-26T05:12:18.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.c.service' 2024-04-26T05:12:18.534 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:18.535 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-26T05:12:18.535 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-26T05:12:18.535 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.b 2024-04-26T05:12:18.857 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:18 smithi195 systemd[1]: Stopping Ceph mon.b for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:19.180 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b[90198]: 2024-04-26T05:12:18.855+0000 7fad22fc9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:19.181 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b[90198]: 2024-04-26T05:12:18.855+0000 7fad22fc9700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:12:19.181 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:18 smithi195 podman[100614]: 2024-04-26 05:12:18.886437407 +0000 UTC m=+0.153848225 container died fc88dd94e9a698dacd0c40eb0c0e2205b4712eabe063507a38bd28360c879953 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:12:19.530 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:19 smithi195 podman[100614]: 2024-04-26 05:12:19.361330086 +0000 UTC m=+0.628740911 container cleanup fc88dd94e9a698dacd0c40eb0c0e2205b4712eabe063507a38bd28360c879953 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T05:12:19.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Apr 26 05:12:19 smithi195 bash[100614]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mon-b 2024-04-26T05:12:19.785 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mon.b.service' 2024-04-26T05:12:19.831 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:19.831 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-26T05:12:19.832 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-26T05:12:19.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y 2024-04-26T05:12:20.107 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:19 smithi152 systemd[1]: Stopping Ceph mgr.y for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:20.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:20 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:12:20.106+0000 7f70392d5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:20.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:20 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y[92644]: 2024-04-26T05:12:20.106+0000 7f70392d5700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-26T05:12:20.380 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:20 smithi152 podman[155685]: 2024-04-26 05:12:20.138119256 +0000 UTC m=+0.121355401 container died ec2ad8342f2fccc0e4fd4cdd56b919e913b22ce973dac8affdc80c41c62dcad1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:12:20.765 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:20 smithi152 podman[155685]: 2024-04-26 05:12:20.538191757 +0000 UTC m=+0.521427901 container cleanup ec2ad8342f2fccc0e4fd4cdd56b919e913b22ce973dac8affdc80c41c62dcad1 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb) 2024-04-26T05:12:20.765 INFO:journalctl@ceph.mgr.y.smithi152.stdout:Apr 26 05:12:20 smithi152 bash[155685]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-y 2024-04-26T05:12:20.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-26T05:12:20.945 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:20.945 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-26T05:12:20.945 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-26T05:12:20.945 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x 2024-04-26T05:12:21.280 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:12:20 smithi195 systemd[1]: Stopping Ceph mgr.x for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:21.558 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:12:21 smithi195 podman[100745]: 2024-04-26 05:12:21.28228747 +0000 UTC m=+0.143898797 container died 178e4f8e339ac29bfe68380b7d5ff251a4aeafc614e6718f4d8fc04ef9904d8b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T05:12:21.920 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:12:21 smithi195 podman[100745]: 2024-04-26 05:12:21.64002417 +0000 UTC m=+0.501635492 container cleanup 178e4f8e339ac29bfe68380b7d5ff251a4aeafc614e6718f4d8fc04ef9904d8b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-26T05:12:21.920 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Apr 26 05:12:21 smithi195 bash[100745]: ceph-164326a4-0388-11ef-bc93-c7b262605968-mgr-x 2024-04-26T05:12:22.047 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-26T05:12:22.079 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:22.080 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-26T05:12:22.080 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-26T05:12:22.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.0 2024-04-26T05:12:22.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:22 smithi152 systemd[1]: Stopping Ceph osd.0 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:22.880 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:22 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[114948]: 2024-04-26T05:12:22.515+0000 7f1791c2a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:22.880 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:22 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[114948]: 2024-04-26T05:12:22.515+0000 7f1791c2a700 -1 osd.0 111 *** Got signal Terminated *** 2024-04-26T05:12:22.880 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:22 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0[114948]: 2024-04-26T05:12:22.515+0000 7f1791c2a700 -1 osd.0 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:12:24.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:24.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:12:24.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:24.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:12:28.128 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:27 smithi152 podman[155817]: 2024-04-26 05:12:27.870287094 +0000 UTC m=+5.603148033 container died 91fec4f30bc46101c87f38757cb12dad3643900a3b9982d6f8ae3c27cd85b275 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:12:28.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:28 smithi152 podman[155817]: 2024-04-26 05:12:28.293041752 +0000 UTC m=+6.025902724 container cleanup 91fec4f30bc46101c87f38757cb12dad3643900a3b9982d6f8ae3c27cd85b275 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T05:12:28.380 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:28 smithi152 bash[155817]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0 2024-04-26T05:12:28.880 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:28 smithi152 podman[155840]: 2024-04-26 05:12:28.486138938 +0000 UTC m=+0.605626424 container remove 91fec4f30bc46101c87f38757cb12dad3643900a3b9982d6f8ae3c27cd85b275 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:12:29.215 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:28 smithi152 podman[155936]: 2024-04-26 05:12:28.81923894 +0000 UTC m=+0.017931842 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:12:29.216 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:28 smithi152 podman[155936]: 2024-04-26 05:12:28.962557758 +0000 UTC m=+0.161250646 container create 73a5e235fb4a677cc90141ac6977da025af64a8f98b07c7591be4cf34918359a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T05:12:29.216 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:29 smithi152 podman[155936]: 2024-04-26 05:12:29.215609593 +0000 UTC m=+0.414302481 container init 73a5e235fb4a677cc90141ac6977da025af64a8f98b07c7591be4cf34918359a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-26T05:12:29.509 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:29 smithi152 podman[155936]: 2024-04-26 05:12:29.22139562 +0000 UTC m=+0.420088517 container start 73a5e235fb4a677cc90141ac6977da025af64a8f98b07c7591be4cf34918359a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb) 2024-04-26T05:12:29.510 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:29 smithi152 podman[155936]: 2024-04-26 05:12:29.307129107 +0000 UTC m=+0.505822006 container attach 73a5e235fb4a677cc90141ac6977da025af64a8f98b07c7591be4cf34918359a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T05:12:29.852 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 26 05:12:29 smithi152 podman[155936]: 2024-04-26 05:12:29.508388679 +0000 UTC m=+0.707081577 container died 73a5e235fb4a677cc90141ac6977da025af64a8f98b07c7591be4cf34918359a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:12:30.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.0.service' 2024-04-26T05:12:30.156 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:30.156 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-26T05:12:30.156 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-26T05:12:30.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.1 2024-04-26T05:12:30.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:30 smithi152 systemd[1]: Stopping Ceph osd.1 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:30.880 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:30 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[120871]: 2024-04-26T05:12:30.574+0000 7f61f3601700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:30.880 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:30 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[120871]: 2024-04-26T05:12:30.574+0000 7f61f3601700 -1 osd.1 111 *** Got signal Terminated *** 2024-04-26T05:12:30.880 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:30 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1[120871]: 2024-04-26T05:12:30.574+0000 7f61f3601700 -1 osd.1 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:12:34.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:34.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:12:34.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:34.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:12:34.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:34 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:34.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:12:36.282 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:35 smithi152 podman[156066]: 2024-04-26 05:12:35.977287255 +0000 UTC m=+5.497681173 container died 5082cc3b9b9ce30f4ee2e0688dae663f5db3502cb1e0f3619715a444496b12d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:12:36.629 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:36 smithi152 podman[156066]: 2024-04-26 05:12:36.489366188 +0000 UTC m=+6.009760148 container cleanup 5082cc3b9b9ce30f4ee2e0688dae663f5db3502cb1e0f3619715a444496b12d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T05:12:36.630 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:36 smithi152 bash[156066]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1 2024-04-26T05:12:37.111 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:36 smithi152 podman[156088]: 2024-04-26 05:12:36.707934206 +0000 UTC m=+0.729175245 container remove 5082cc3b9b9ce30f4ee2e0688dae663f5db3502cb1e0f3619715a444496b12d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:12:37.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:37 smithi152 podman[156188]: 2024-04-26 05:12:37.014229541 +0000 UTC m=+0.018309436 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:12:37.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:37 smithi152 podman[156188]: 2024-04-26 05:12:37.117143253 +0000 UTC m=+0.121223180 container create 9c177d6301e81caa6e307f94a35e951c9bb01c8639d733f39971187dfb3ca45b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T05:12:37.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:37 smithi152 podman[156188]: 2024-04-26 05:12:37.303689422 +0000 UTC m=+0.307769309 container init 9c177d6301e81caa6e307f94a35e951c9bb01c8639d733f39971187dfb3ca45b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:12:37.380 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:37 smithi152 podman[156188]: 2024-04-26 05:12:37.309121937 +0000 UTC m=+0.313201822 container start 9c177d6301e81caa6e307f94a35e951c9bb01c8639d733f39971187dfb3ca45b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-26T05:12:37.880 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:37 smithi152 podman[156188]: 2024-04-26 05:12:37.386568885 +0000 UTC m=+0.390648772 container attach 9c177d6301e81caa6e307f94a35e951c9bb01c8639d733f39971187dfb3ca45b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T05:12:37.880 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 26 05:12:37 smithi152 podman[156188]: 2024-04-26 05:12:37.604789606 +0000 UTC m=+0.608869494 container died 9c177d6301e81caa6e307f94a35e951c9bb01c8639d733f39971187dfb3ca45b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T05:12:38.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.1.service' 2024-04-26T05:12:38.486 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:38.486 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-26T05:12:38.486 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-26T05:12:38.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.2 2024-04-26T05:12:38.713 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:38 smithi152 systemd[1]: Stopping Ceph osd.2 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:39.129 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:38 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[126138]: 2024-04-26T05:12:38.821+0000 7f0332ac2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:39.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:38 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[126138]: 2024-04-26T05:12:38.821+0000 7f0332ac2700 -1 osd.2 111 *** Got signal Terminated *** 2024-04-26T05:12:39.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:38 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2[126138]: 2024-04-26T05:12:38.821+0000 7f0332ac2700 -1 osd.2 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:12:44.545 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:44 smithi152 podman[156321]: 2024-04-26 05:12:44.346191901 +0000 UTC m=+5.635916006 container died ec85e9f04bc66b15115beb0c708280552c1318b6221331710ce461d0e2e27761 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:12:44.546 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:44.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:12:44.546 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:44.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:12:44.546 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:44 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:44.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:12:44.880 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:44 smithi152 podman[156321]: 2024-04-26 05:12:44.75289659 +0000 UTC m=+6.042620695 container cleanup ec85e9f04bc66b15115beb0c708280552c1318b6221331710ce461d0e2e27761 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-26T05:12:44.881 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:44 smithi152 bash[156321]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2 2024-04-26T05:12:45.369 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:44 smithi152 podman[156353]: 2024-04-26 05:12:44.963015718 +0000 UTC m=+0.613960405 container remove ec85e9f04bc66b15115beb0c708280552c1318b6221331710ce461d0e2e27761 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b22e2eb) 2024-04-26T05:12:45.630 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:45 smithi152 podman[156449]: 2024-04-26 05:12:45.271642282 +0000 UTC m=+0.018640293 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:12:45.630 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:45 smithi152 podman[156449]: 2024-04-26 05:12:45.406556729 +0000 UTC m=+0.153554729 container create ded899346143424b6f0c58c8715929b09c7ded851f7dba8f890b98a4ce6e64e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:12:45.630 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:45 smithi152 podman[156449]: 2024-04-26 05:12:45.608596609 +0000 UTC m=+0.355594621 container init ded899346143424b6f0c58c8715929b09c7ded851f7dba8f890b98a4ce6e64e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T05:12:45.631 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:45 smithi152 podman[156449]: 2024-04-26 05:12:45.614492095 +0000 UTC m=+0.361490097 container start ded899346143424b6f0c58c8715929b09c7ded851f7dba8f890b98a4ce6e64e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T05:12:46.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:45 smithi152 podman[156449]: 2024-04-26 05:12:45.69981823 +0000 UTC m=+0.446816242 container attach ded899346143424b6f0c58c8715929b09c7ded851f7dba8f890b98a4ce6e64e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T05:12:46.130 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 26 05:12:45 smithi152 podman[156449]: 2024-04-26 05:12:45.904529895 +0000 UTC m=+0.651527897 container died ded899346143424b6f0c58c8715929b09c7ded851f7dba8f890b98a4ce6e64e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True) 2024-04-26T05:12:46.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.2.service' 2024-04-26T05:12:46.656 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:46.656 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-26T05:12:46.656 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-26T05:12:46.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.3 2024-04-26T05:12:46.880 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:46 smithi152 systemd[1]: Stopping Ceph osd.3 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:47.380 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:12:47.001+0000 7f1d47acd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:47.380 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:12:47.001+0000 7f1d47acd700 -1 osd.3 111 *** Got signal Terminated *** 2024-04-26T05:12:47.380 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:47 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:12:47.001+0000 7f1d47acd700 -1 osd.3 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:12:50.380 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:50 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:12:50.010+0000 7f1d400b9700 -1 osd.3 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.596383+0000 front 2024-04-26T05:12:25.596401+0000 (oldest deadline 2024-04-26T05:12:49.096512+0000) 2024-04-26T05:12:50.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:50.118+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:50.788 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:50 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:50.692+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:51.379 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:51 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3[131574]: 2024-04-26T05:12:51.017+0000 7f1d400b9700 -1 osd.3 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.596383+0000 front 2024-04-26T05:12:25.596401+0000 (oldest deadline 2024-04-26T05:12:49.096512+0000) 2024-04-26T05:12:51.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:51.123+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:52.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:51 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:51.662+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:52.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:52 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:52.161+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:52.630 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:52 smithi152 podman[156579]: 2024-04-26 05:12:52.354577584 +0000 UTC m=+5.479577321 container died 41cf15485f589acde56b3fde44cb5d2afc777b96fecf9831bcb8434c628ace9c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:12:53.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:52 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:52.700+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:53.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:52 smithi152 podman[156579]: 2024-04-26 05:12:52.765569749 +0000 UTC m=+5.890569495 container cleanup 41cf15485f589acde56b3fde44cb5d2afc777b96fecf9831bcb8434c628ace9c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:12:53.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:52 smithi152 bash[156579]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3 2024-04-26T05:12:53.130 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:52 smithi152 podman[156601]: 2024-04-26 05:12:52.950255469 +0000 UTC m=+0.589549157 container remove 41cf15485f589acde56b3fde44cb5d2afc777b96fecf9831bcb8434c628ace9c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0) 2024-04-26T05:12:53.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:53.290+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.509667+0000 front 2024-04-26T05:12:25.509718+0000 (oldest deadline 2024-04-26T05:12:52.509873+0000) 2024-04-26T05:12:53.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:12:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:12:53.216+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:12:53.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:53.137+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:53.587 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:53 smithi152 podman[156696]: 2024-04-26 05:12:53.181092242 +0000 UTC m=+0.018106460 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:12:53.587 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:53 smithi152 podman[156696]: 2024-04-26 05:12:53.284738661 +0000 UTC m=+0.121752907 container create 5c965c0f230b4af337d8683041f261046e072df9e78a39750680990598790f99 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T05:12:53.587 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:53 smithi152 podman[156696]: 2024-04-26 05:12:53.49567069 +0000 UTC m=+0.332684901 container init 5c965c0f230b4af337d8683041f261046e072df9e78a39750680990598790f99 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T05:12:53.587 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:53 smithi152 podman[156696]: 2024-04-26 05:12:53.501434954 +0000 UTC m=+0.338449158 container start 5c965c0f230b4af337d8683041f261046e072df9e78a39750680990598790f99 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T05:12:53.588 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:53 smithi152 podman[156696]: 2024-04-26 05:12:53.586958994 +0000 UTC m=+0.423973198 container attach 5c965c0f230b4af337d8683041f261046e072df9e78a39750680990598790f99 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:12:53.880 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:53 smithi152 podman[156696]: 2024-04-26 05:12:53.792093678 +0000 UTC m=+0.629107879 container died 5c965c0f230b4af337d8683041f261046e072df9e78a39750680990598790f99 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:12:54.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:53 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:53.737+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:54.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.3.service' 2024-04-26T05:12:54.416 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:54 smithi152 podman[156742]: 2024-04-26 05:12:54.315315874 +0000 UTC m=+0.517479047 container remove 5c965c0f230b4af337d8683041f261046e072df9e78a39750680990598790f99 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T05:12:54.416 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:54 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T05:12:54.417 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:54 smithi152 systemd[1]: Stopped Ceph osd.3 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:12:54.417 INFO:journalctl@ceph.osd.3.smithi152.stdout:Apr 26 05:12:54 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.3.service: Consumed 3.555s CPU time. 2024-04-26T05:12:54.417 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:54.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:12:54.417 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:54.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:12:54.417 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:12:54 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:12:54.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:12:54.453 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:54.453 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-26T05:12:54.453 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-26T05:12:54.453 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.4 2024-04-26T05:12:54.482 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:54.241+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.509667+0000 front 2024-04-26T05:12:25.509718+0000 (oldest deadline 2024-04-26T05:12:52.509873+0000) 2024-04-26T05:12:54.482 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:54.128+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:54.483 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:12:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:12:54.260+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:12:54.773 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:54 smithi195 systemd[1]: Stopping Ceph osd.4 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:55.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:54.771+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:55.031 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:54.993+0000 7f71f3add700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:12:55.031 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:54.993+0000 7f71f3add700 -1 osd.4 111 *** Got signal Terminated *** 2024-04-26T05:12:55.031 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:54 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:54.993+0000 7f71f3add700 -1 osd.4 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:12:55.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:12:55 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:12:55.272+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:12:55.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:55 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:55.243+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.509667+0000 front 2024-04-26T05:12:25.509718+0000 (oldest deadline 2024-04-26T05:12:52.509873+0000) 2024-04-26T05:12:55.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:55 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:55.175+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:56.138 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:55 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:55.799+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:56.530 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:56 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:56.276+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.509667+0000 front 2024-04-26T05:12:25.509718+0000 (oldest deadline 2024-04-26T05:12:52.509873+0000) 2024-04-26T05:12:56.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:12:56 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:12:56.234+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:12:56.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:56 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:56.137+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:57.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:56 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:56.762+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:57.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:12:57 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:12:57.218+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:12:57.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:57 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:57.129+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:57.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:57 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:57.314+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.509667+0000 front 2024-04-26T05:12:25.509718+0000 (oldest deadline 2024-04-26T05:12:52.509873+0000) 2024-04-26T05:12:57.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:57 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:57.314+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.311299+0000 front 2024-04-26T05:12:35.311396+0000 (oldest deadline 2024-04-26T05:12:56.411358+0000) 2024-04-26T05:12:58.092 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:57 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:57.805+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:58.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:12:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:12:58.260+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:12:58.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:58.295+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.509667+0000 front 2024-04-26T05:12:25.509718+0000 (oldest deadline 2024-04-26T05:12:52.509873+0000) 2024-04-26T05:12:58.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:58.295+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.311299+0000 front 2024-04-26T05:12:35.311396+0000 (oldest deadline 2024-04-26T05:12:56.411358+0000) 2024-04-26T05:12:58.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:58.090+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:58.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:58.091+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:12:59.122 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:58 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:58.798+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:12:59.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:12:59 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:12:59.258+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:12:59.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:59 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:59.121+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:12:59.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:12:59 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:12:59.121+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:12:59.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:59 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:59.322+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:25.509667+0000 front 2024-04-26T05:12:25.509718+0000 (oldest deadline 2024-04-26T05:12:52.509873+0000) 2024-04-26T05:12:59.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:12:59 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4[36886]: 2024-04-26T05:12:59.322+0000 7f71ef0d9700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.311299+0000 front 2024-04-26T05:12:35.311396+0000 (oldest deadline 2024-04-26T05:12:56.411358+0000) 2024-04-26T05:13:00.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:12:59 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:12:59.756+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:00.531 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:00 smithi195 podman[100890]: 2024-04-26 05:13:00.351221658 +0000 UTC m=+5.693036426 container died 958de1ceb55129cb020c187d0b0f35b24b8481aefbe6c50b5470850bb00a4e91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T05:13:00.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:00.300+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:00.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:00.300+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:00.532 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:00.100+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:00.532 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:00.100+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:00.894 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:00.749+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:00.894 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:00 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:00.749+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:00.895 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:00 smithi195 podman[100890]: 2024-04-26 05:13:00.699989603 +0000 UTC m=+6.041804360 container cleanup 958de1ceb55129cb020c187d0b0f35b24b8481aefbe6c50b5470850bb00a4e91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T05:13:00.895 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:00 smithi195 bash[100890]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4 2024-04-26T05:13:01.214 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:00 smithi195 podman[100912]: 2024-04-26 05:13:00.892968101 +0000 UTC m=+0.532546714 container remove 958de1ceb55129cb020c187d0b0f35b24b8481aefbe6c50b5470850bb00a4e91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-26T05:13:01.214 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:01.110+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:01.215 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:01.110+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:01.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:01.333+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:01.526 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:01.333+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:01.526 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:01 smithi195 podman[101011]: 2024-04-26 05:13:01.220658581 +0000 UTC m=+0.019676551 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:13:01.526 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:01 smithi195 podman[101011]: 2024-04-26 05:13:01.321627451 +0000 UTC m=+0.120645444 container create 8a858fb96a2186abcd2462c944f95c499cc135aba9b42f096d157afe2f54912f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:13:01.527 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:01.424Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-26T05:13:01.527 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:01.427Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-26T05:13:01.527 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:01.427Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-26T05:13:01.527 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:01.429Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-26T05:13:01.527 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:01.429Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-26T05:13:01.778 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:01.704+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:01.778 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:01 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:01.704+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:01.779 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:01 smithi195 podman[101011]: 2024-04-26 05:13:01.525177527 +0000 UTC m=+0.324195478 container init 8a858fb96a2186abcd2462c944f95c499cc135aba9b42f096d157afe2f54912f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:13:01.779 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:01 smithi195 podman[101011]: 2024-04-26 05:13:01.53135842 +0000 UTC m=+0.330376372 container start 8a858fb96a2186abcd2462c944f95c499cc135aba9b42f096d157afe2f54912f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:13:01.779 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:01 smithi195 podman[101011]: 2024-04-26 05:13:01.615847005 +0000 UTC m=+0.414864971 container attach 8a858fb96a2186abcd2462c944f95c499cc135aba9b42f096d157afe2f54912f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=quincy-c61472c, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T05:13:02.030 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:01 smithi195 podman[101011]: 2024-04-26 05:13:01.828102504 +0000 UTC m=+0.627120458 container died 8a858fb96a2186abcd2462c944f95c499cc135aba9b42f096d157afe2f54912f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-04-26T05:13:02.329 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.4.service' 2024-04-26T05:13:02.391 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:02.331+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:02.392 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:02.331+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:02.392 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:02 smithi195 podman[101057]: 2024-04-26 05:13:02.310787158 +0000 UTC m=+0.477811735 container remove 8a858fb96a2186abcd2462c944f95c499cc135aba9b42f096d157afe2f54912f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T05:13:02.392 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:02 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-26T05:13:02.392 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:02 smithi195 systemd[1]: Stopped Ceph osd.4 for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:13:02.392 INFO:journalctl@ceph.osd.4.smithi195.stdout:Apr 26 05:13:02 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.4.service: Consumed 8.386s CPU time. 2024-04-26T05:13:02.393 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:02.159+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:02.393 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:02.159+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:02.407 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:13:02.407 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-26T05:13:02.408 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-26T05:13:02.408 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.5 2024-04-26T05:13:02.660 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:02 smithi195 systemd[1]: Stopping Ceph osd.5 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:13:02.660 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:02.658+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:02.660 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:02.658+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:03.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:02.741+0000 7f41f246f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:13:03.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:02.741+0000 7f41f246f700 -1 osd.5 111 *** Got signal Terminated *** 2024-04-26T05:13:03.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:02 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:02.741+0000 7f41f246f700 -1 osd.5 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:13:03.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:03 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:03.305+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:03.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:03 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:03.305+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:03.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:03 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:03.196+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:03.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:03 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:03.196+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:04.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:03 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:03.646+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:04.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:03 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:03.646+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:04.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:04.281+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:04.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:04.281+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:04.532 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:04.167+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:04.532 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:04.167+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:04.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:04.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:13:04.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:04.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:13:04.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:04 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:04.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:13:05.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:04.684+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:05.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:04 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:04.684+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:05.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:05.318+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:05.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:05.318+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:05.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:05.318+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:05.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:05.165+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:05.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:05.165+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:06.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:05.646+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:06.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:05.646+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:06.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:05 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:05.646+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:40.837711+0000 front 2024-04-26T05:12:40.837684+0000 (oldest deadline 2024-04-26T05:13:05.537635+0000) 2024-04-26T05:13:06.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:06.159+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:06.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:06.159+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:06.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:06.364+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:06.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:06.364+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:06.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:06.364+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:07.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:06.630+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:07.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:06.630+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:07.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:06 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:06.630+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:40.837711+0000 front 2024-04-26T05:12:40.837684+0000 (oldest deadline 2024-04-26T05:13:05.537635+0000) 2024-04-26T05:13:07.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:07.398+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:07.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:07.398+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:07.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:07.398+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:07.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:07.193+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:07.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:07.193+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:08.030 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:07.638+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:27.035298+0000 front 2024-04-26T05:12:27.035337+0000 (oldest deadline 2024-04-26T05:12:49.935106+0000) 2024-04-26T05:13:08.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:07.638+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:35.136452+0000 front 2024-04-26T05:12:35.136539+0000 (oldest deadline 2024-04-26T05:12:59.836378+0000) 2024-04-26T05:13:08.031 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:07 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5[42926]: 2024-04-26T05:13:07.638+0000 7f41eba67700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:40.837711+0000 front 2024-04-26T05:12:40.837684+0000 (oldest deadline 2024-04-26T05:13:05.537635+0000) 2024-04-26T05:13:08.407 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:08 smithi195 podman[101141]: 2024-04-26 05:13:08.117574994 +0000 UTC m=+5.491171236 container died 5a97e623bdc9e4ab1608f47f5201face7acac57c98918d4a77404c46a8d0cf71 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T05:13:08.407 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:08.232+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:08.407 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:08.232+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:08.780 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:08.406+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:08.781 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:08.406+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:08.781 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:08 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:08.406+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:08.781 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:08 smithi195 podman[101141]: 2024-04-26 05:13:08.438513114 +0000 UTC m=+5.812109410 container cleanup 5a97e623bdc9e4ab1608f47f5201face7acac57c98918d4a77404c46a8d0cf71 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=) 2024-04-26T05:13:08.781 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:08 smithi195 bash[101141]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5 2024-04-26T05:13:08.782 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:08 smithi195 podman[101165]: 2024-04-26 05:13:08.639807133 +0000 UTC m=+0.518676258 container remove 5a97e623bdc9e4ab1608f47f5201face7acac57c98918d4a77404c46a8d0cf71 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-26T05:13:09.381 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:09.209+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:09.381 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:09.209+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:09.381 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:09.209+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:09.381 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:09 smithi195 podman[101264]: 2024-04-26 05:13:08.967298559 +0000 UTC m=+0.018257544 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:13:09.381 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:09 smithi195 podman[101264]: 2024-04-26 05:13:09.073993092 +0000 UTC m=+0.124952087 container create 82500c55091da35e00a020d73dcd49b8fef26aa575cbe1ff582522dfb310503f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True) 2024-04-26T05:13:09.382 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:09 smithi195 podman[101264]: 2024-04-26 05:13:09.301788137 +0000 UTC m=+0.352747103 container init 82500c55091da35e00a020d73dcd49b8fef26aa575cbe1ff582522dfb310503f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:13:09.382 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:09 smithi195 podman[101264]: 2024-04-26 05:13:09.30761478 +0000 UTC m=+0.358573750 container start 82500c55091da35e00a020d73dcd49b8fef26aa575cbe1ff582522dfb310503f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-26T05:13:09.781 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:09 smithi195 podman[101264]: 2024-04-26 05:13:09.420119002 +0000 UTC m=+0.471077967 container attach 82500c55091da35e00a020d73dcd49b8fef26aa575cbe1ff582522dfb310503f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T05:13:09.781 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:09 smithi195 podman[101264]: 2024-04-26 05:13:09.589633173 +0000 UTC m=+0.640592128 container died 82500c55091da35e00a020d73dcd49b8fef26aa575cbe1ff582522dfb310503f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:13:09.781 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:09.379+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:09.781 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:09.379+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:09.781 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:09 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:09.379+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:10.136 INFO:journalctl@ceph.osd.5.smithi195.stdout:Apr 26 05:13:10 smithi195 podman[101310]: 2024-04-26 05:13:10.121550762 +0000 UTC m=+0.526094977 container remove 82500c55091da35e00a020d73dcd49b8fef26aa575cbe1ff582522dfb310503f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-26T05:13:10.141 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.5.service' 2024-04-26T05:13:10.177 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:13:10.177 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-26T05:13:10.178 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-26T05:13:10.178 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.6 2024-04-26T05:13:10.494 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:10.177+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:10.495 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:10.177+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:10.495 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:10.177+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:10.495 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:10 smithi195 systemd[1]: Stopping Ceph osd.6 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:13:10.495 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:10.382+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:10.495 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:10.382+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:10.496 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:10.382+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:10.496 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:10.493+0000 7fe466dd5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:13:10.496 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:10.493+0000 7fe466dd5700 -1 osd.6 111 *** Got signal Terminated *** 2024-04-26T05:13:10.496 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:10 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:10.493+0000 7fe466dd5700 -1 osd.6 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:13:11.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:11 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:11.335+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:11.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:11 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:11.335+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:11.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:11 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:11.335+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:11.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:11 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:11.132+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:11.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:11 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:11.132+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:11.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:11 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:11.132+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:12.358 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:12.101+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:12.359 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:12.101+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:12.359 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:12.101+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:12.780 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:12.357+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:12.781 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:12.357+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:12.781 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:12 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:12.357+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:13.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:13.322+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:13.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:13.322+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:13.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:13.322+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:13.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:13.135+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:13.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:13.135+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:13.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:13 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:13.135+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:14.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:14.358+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:14.531 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:14.358+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:14.532 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:14.358+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:14.532 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:14.123+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:14.532 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:14.123+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:14.533 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:14 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:14.123+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:14.540 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:14.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:13:14.540 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:14.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:13:14.540 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:14 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:14.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:13:15.533 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:15.133+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:15.534 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:15.133+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:15.534 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:15.133+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:15.534 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:15.345+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:26.578587+0000 front 2024-04-26T05:12:26.578488+0000 (oldest deadline 2024-04-26T05:12:52.478272+0000) 2024-04-26T05:13:15.534 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:15.345+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:33.579414+0000 front 2024-04-26T05:12:33.579315+0000 (oldest deadline 2024-04-26T05:12:59.379796+0000) 2024-04-26T05:13:15.534 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:15.345+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:39.380175+0000 front 2024-04-26T05:12:39.380240+0000 (oldest deadline 2024-04-26T05:13:05.280340+0000) 2024-04-26T05:13:15.534 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:15 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6[48923]: 2024-04-26T05:13:15.345+0000 7fe464bd6700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.152:6830 osd.3 since back 2024-04-26T05:12:50.481956+0000 front 2024-04-26T05:12:50.481982+0000 (oldest deadline 2024-04-26T05:13:15.181849+0000) 2024-04-26T05:13:16.156 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:15 smithi195 podman[101396]: 2024-04-26 05:13:15.905977034 +0000 UTC m=+5.507241127 container died f77aacbd5fd57f0ca24c4c89e66698c88c6c41522fcc18c7bef8ec942abbc258 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T05:13:16.156 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:16 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:16.155+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:16.157 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:16 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:16.155+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:16.157 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:16 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:16.155+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:16.425 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:16 smithi195 podman[101396]: 2024-04-26 05:13:16.265221264 +0000 UTC m=+5.866485345 container cleanup f77aacbd5fd57f0ca24c4c89e66698c88c6c41522fcc18c7bef8ec942abbc258 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:13:16.426 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:16 smithi195 bash[101396]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6 2024-04-26T05:13:16.780 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:16 smithi195 podman[101419]: 2024-04-26 05:13:16.424867504 +0000 UTC m=+0.511856292 container remove f77aacbd5fd57f0ca24c4c89e66698c88c6c41522fcc18c7bef8ec942abbc258 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:13:17.095 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:16 smithi195 podman[101518]: 2024-04-26 05:13:16.817337759 +0000 UTC m=+0.102596819 container create 7653ef1f18af2a7734d8dce4c78c06477837678784dfd6dc90de07f87a607dd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-26T05:13:17.095 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:16 smithi195 podman[101518]: 2024-04-26 05:13:16.732777615 +0000 UTC m=+0.018036703 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:13:17.095 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:17 smithi195 podman[101518]: 2024-04-26 05:13:17.011326068 +0000 UTC m=+0.296585126 container init 7653ef1f18af2a7734d8dce4c78c06477837678784dfd6dc90de07f87a607dd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=quincy-c61472c, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T05:13:17.095 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:17 smithi195 podman[101518]: 2024-04-26 05:13:17.017321246 +0000 UTC m=+0.302580311 container start 7653ef1f18af2a7734d8dce4c78c06477837678784dfd6dc90de07f87a607dd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:13:17.095 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:17 smithi195 podman[101518]: 2024-04-26 05:13:17.094728756 +0000 UTC m=+0.379987818 container attach 7653ef1f18af2a7734d8dce4c78c06477837678784dfd6dc90de07f87a607dd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:13:17.530 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:17 smithi195 podman[101518]: 2024-04-26 05:13:17.300815643 +0000 UTC m=+0.586074704 container died 7653ef1f18af2a7734d8dce4c78c06477837678784dfd6dc90de07f87a607dd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c) 2024-04-26T05:13:17.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:17.201+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:17.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:17.201+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:17.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:17.201+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:17.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:17 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:17.201+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6830 osd.3 since back 2024-04-26T05:12:51.120848+0000 front 2024-04-26T05:12:51.120795+0000 (oldest deadline 2024-04-26T05:13:17.020647+0000) 2024-04-26T05:13:17.843 INFO:journalctl@ceph.osd.6.smithi195.stdout:Apr 26 05:13:17 smithi195 podman[101564]: 2024-04-26 05:13:17.839851696 +0000 UTC m=+0.533275573 container remove 7653ef1f18af2a7734d8dce4c78c06477837678784dfd6dc90de07f87a607dd5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:13:17.862 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.6.service' 2024-04-26T05:13:17.898 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:13:17.898 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-26T05:13:17.898 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-26T05:13:17.899 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.7 2024-04-26T05:13:18.184 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:17 smithi195 systemd[1]: Stopping Ceph osd.7 for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:13:18.184 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:18.182+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:18.184 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:18.182+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:18.184 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:18.182+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:18.185 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:18.182+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6830 osd.3 since back 2024-04-26T05:12:51.120848+0000 front 2024-04-26T05:12:51.120795+0000 (oldest deadline 2024-04-26T05:13:17.020647+0000) 2024-04-26T05:13:18.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:18.223+0000 7fa0a2876700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:13:18.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:18.223+0000 7fa0a2876700 -1 osd.7 111 *** Got signal Terminated *** 2024-04-26T05:13:18.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:18 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:18.223+0000 7fa0a2876700 -1 osd.7 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:13:19.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:19 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:19.164+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:19.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:19 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:19.164+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:19.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:19 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:19.164+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:19.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:19 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:19.164+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6830 osd.3 since back 2024-04-26T05:12:51.120848+0000 front 2024-04-26T05:12:51.120795+0000 (oldest deadline 2024-04-26T05:13:17.020647+0000) 2024-04-26T05:13:20.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:20.206+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:20.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:20.206+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:20.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:20.206+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:20.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:20 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:20.206+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6830 osd.3 since back 2024-04-26T05:12:51.120848+0000 front 2024-04-26T05:12:51.120795+0000 (oldest deadline 2024-04-26T05:13:17.020647+0000) 2024-04-26T05:13:21.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:21.225+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:21.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:21.225+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:21.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:21.225+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:21.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:21 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:21.225+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6830 osd.3 since back 2024-04-26T05:12:51.120848+0000 front 2024-04-26T05:12:51.120795+0000 (oldest deadline 2024-04-26T05:13:17.020647+0000) 2024-04-26T05:13:22.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:22.257+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6806 osd.0 since back 2024-04-26T05:12:24.416862+0000 front 2024-04-26T05:12:24.416938+0000 (oldest deadline 2024-04-26T05:12:49.716887+0000) 2024-04-26T05:13:22.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:22.257+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6814 osd.1 since back 2024-04-26T05:12:34.318242+0000 front 2024-04-26T05:12:34.318301+0000 (oldest deadline 2024-04-26T05:12:57.218166+0000) 2024-04-26T05:13:22.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:22.257+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6822 osd.2 since back 2024-04-26T05:12:43.520093+0000 front 2024-04-26T05:12:43.520109+0000 (oldest deadline 2024-04-26T05:13:08.819882+0000) 2024-04-26T05:13:22.531 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:22 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7[54984]: 2024-04-26T05:13:22.257+0000 7fa09de72700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.152:6830 osd.3 since back 2024-04-26T05:12:51.120848+0000 front 2024-04-26T05:12:51.120795+0000 (oldest deadline 2024-04-26T05:13:17.020647+0000) 2024-04-26T05:13:23.895 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:23 smithi195 podman[101648]: 2024-04-26 05:13:23.607354709 +0000 UTC m=+5.495709276 container died e1529e54096a9f14a36a28e2520500855b2b2addb55be5cb31afe0f1f07db258 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T05:13:24.227 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:23 smithi195 podman[101648]: 2024-04-26 05:13:23.975628553 +0000 UTC m=+5.863983123 container cleanup e1529e54096a9f14a36a28e2520500855b2b2addb55be5cb31afe0f1f07db258 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-26T05:13:24.228 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:23 smithi195 bash[101648]: ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7 2024-04-26T05:13:24.530 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:24 smithi195 podman[101683]: 2024-04-26 05:13:24.227156701 +0000 UTC m=+0.606929858 container remove e1529e54096a9f14a36a28e2520500855b2b2addb55be5cb31afe0f1f07db258 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:13:24.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:24.284Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:13:24.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:24.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-26T05:13:24.630 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:24 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:24.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-04-26T05:13:24.839 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:24 smithi195 podman[101779]: 2024-04-26 05:13:24.451265715 +0000 UTC m=+0.018165046 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:13:24.839 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:24 smithi195 podman[101779]: 2024-04-26 05:13:24.552974296 +0000 UTC m=+0.119873611 container create 21f16281b5aedf5c2d2352bddeaca7f55d3ee1c7cd797e8d357a6a81233c1655 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, RELEASE=quincy-c61472c) 2024-04-26T05:13:24.839 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:24 smithi195 podman[101779]: 2024-04-26 05:13:24.747134929 +0000 UTC m=+0.314034244 container init 21f16281b5aedf5c2d2352bddeaca7f55d3ee1c7cd797e8d357a6a81233c1655 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:13:24.839 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:24 smithi195 podman[101779]: 2024-04-26 05:13:24.752724196 +0000 UTC m=+0.319623517 container start 21f16281b5aedf5c2d2352bddeaca7f55d3ee1c7cd797e8d357a6a81233c1655 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T05:13:24.839 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:24 smithi195 podman[101779]: 2024-04-26 05:13:24.838470452 +0000 UTC m=+0.405369771 container attach 21f16281b5aedf5c2d2352bddeaca7f55d3ee1c7cd797e8d357a6a81233c1655 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:13:25.281 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:25 smithi195 podman[101779]: 2024-04-26 05:13:25.034324428 +0000 UTC m=+0.601223742 container died 21f16281b5aedf5c2d2352bddeaca7f55d3ee1c7cd797e8d357a6a81233c1655 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, ceph=True, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-26T05:13:25.636 INFO:journalctl@ceph.osd.7.smithi195.stdout:Apr 26 05:13:25 smithi195 podman[101825]: 2024-04-26 05:13:25.616480426 +0000 UTC m=+0.576248692 container remove 21f16281b5aedf5c2d2352bddeaca7f55d3ee1c7cd797e8d357a6a81233c1655 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:13:25.641 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@osd.7.service' 2024-04-26T05:13:25.677 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:13:25.677 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-26T05:13:25.677 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-26T05:13:25.678 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a 2024-04-26T05:13:25.997 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 systemd[1]: Stopping Ceph prometheus.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:13:26.262 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:13:26.263 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T05:13:26.263 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T05:13:26.263 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T05:13:26.263 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T05:13:26.263 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T05:13:26.263 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T05:13:26.263 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T05:13:26.264 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T05:13:26.264 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T05:13:26.264 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T05:13:26.264 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:25 smithi195 ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a[79432]: ts=2024-04-26T05:13:25.997Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T05:13:26.264 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:26 smithi195 podman[101909]: 2024-04-26 05:13:26.007688034 +0000 UTC m=+0.118666688 container died 153c7fab44cf92f6a35e7c32d3b845a879f0eab11d7f6c65975590f2e79fa36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:26.530 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:26 smithi195 podman[101909]: 2024-04-26 05:13:26.352959385 +0000 UTC m=+0.463938090 container cleanup 153c7fab44cf92f6a35e7c32d3b845a879f0eab11d7f6c65975590f2e79fa36e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:26.531 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Apr 26 05:13:26 smithi195 bash[101909]: ceph-164326a4-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T05:13:26.777 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-164326a4-0388-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-26T05:13:26.810 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:13:26.811 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-26T05:13:26.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 164326a4-0388-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:13:28.536 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:28 smithi152 systemd[1]: Stopping Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:13:28.851 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:28 smithi152 ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a[91526]: ts=2024-04-26T05:13:28.535Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:13:28.851 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:28 smithi152 podman[157129]: 2024-04-26 05:13:28.57652894 +0000 UTC m=+0.135932866 container died 252d0e292dd9f588a86941640a745a9677ff93dde8e180486d064a8f539d92d7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:29.379 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:29 smithi152 podman[157129]: 2024-04-26 05:13:29.116807631 +0000 UTC m=+0.676211556 container cleanup 252d0e292dd9f588a86941640a745a9677ff93dde8e180486d064a8f539d92d7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:29.380 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:29 smithi152 bash[157129]: ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T05:13:29.380 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:29 smithi152 podman[157149]: 2024-04-26 05:13:29.284795339 +0000 UTC m=+0.705760045 container remove 252d0e292dd9f588a86941640a745a9677ff93dde8e180486d064a8f539d92d7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:29.380 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:29 smithi152 podman[157149]: 2024-04-26 05:13:29.343712907 +0000 UTC m=+0.764677568 volume remove 8ec13b9675f522859ba8f96f8925264ede50a0f2807ba9f217850ced27ca101e 2024-04-26T05:13:29.880 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:29 smithi152 systemd[1]: Stopping Ceph node-exporter.a for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:13:29.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:29 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T05:13:29.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:29 smithi152 systemd[1]: Stopped Ceph alertmanager.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:13:29.880 INFO:journalctl@ceph.alertmanager.a.smithi152.stdout:Apr 26 05:13:29 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 2.076s CPU time. 2024-04-26T05:13:30.306 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 podman[157259]: 2024-04-26 05:13:30.018989885 +0000 UTC m=+0.118382197 container died 8d4a4fd1dbaef8625c7daddeab48d0530c02a7b8eb6cc32cfb325df8740bcc4e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:30.629 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 podman[157259]: 2024-04-26 05:13:30.405461855 +0000 UTC m=+0.504854168 container cleanup 8d4a4fd1dbaef8625c7daddeab48d0530c02a7b8eb6cc32cfb325df8740bcc4e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:30.630 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 bash[157259]: ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T05:13:30.947 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 podman[157278]: 2024-04-26 05:13:30.631520089 +0000 UTC m=+0.617375433 container remove 8d4a4fd1dbaef8625c7daddeab48d0530c02a7b8eb6cc32cfb325df8740bcc4e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:13:30.947 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:13:30.947 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T05:13:30.947 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 systemd[1]: Stopped Ceph node-exporter.a for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:13:30.947 INFO:journalctl@ceph.node-exporter.a.smithi152.stdout:Apr 26 05:13:30 smithi152 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.481s CPU time. 2024-04-26T05:13:41.058 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 164326a4-0388-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:13:42.944 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:42 smithi195 systemd[1]: Stopping Ceph node-exporter.b for 164326a4-0388-11ef-bc93-c7b262605968... 2024-04-26T05:13:43.264 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:42 smithi195 podman[102363]: 2024-04-26 05:13:42.944275114 +0000 UTC m=+0.118511352 container died bdec586e73b34435bc45efb2253133bb2c1ed589d57c45251dbc1ca29ffcc5b0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:13:43.264 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:43 smithi195 podman[102363]: 2024-04-26 05:13:43.263974169 +0000 UTC m=+0.438210404 container cleanup bdec586e73b34435bc45efb2253133bb2c1ed589d57c45251dbc1ca29ffcc5b0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:13:43.530 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:43 smithi195 bash[102363]: ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T05:13:43.531 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:43 smithi195 podman[102382]: 2024-04-26 05:13:43.448314118 +0000 UTC m=+0.508198874 container remove bdec586e73b34435bc45efb2253133bb2c1ed589d57c45251dbc1ca29ffcc5b0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-164326a4-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:13:43.531 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:43 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:13:44.030 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:43 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T05:13:44.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:43 smithi195 systemd[1]: Stopped Ceph node-exporter.b for 164326a4-0388-11ef-bc93-c7b262605968. 2024-04-26T05:13:44.031 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Apr 26 05:13:43 smithi195 systemd[1]: ceph-164326a4-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.379s CPU time. 2024-04-26T05:13:49.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:13:49.594 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:13:49.620 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-26T05:13:49.622 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595/remote/smithi152/crash 2024-04-26T05:13:49.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:13:49.675 INFO:teuthology.orchestra.run.smithi152.stderr:tar: /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:13:49.675 INFO:teuthology.orchestra.run.smithi152.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:13:49.677 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595/remote/smithi195/crash 2024-04-26T05:13:49.678 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:13:49.717 INFO:teuthology.orchestra.run.smithi195.stderr:tar: /var/lib/ceph/164326a4-0388-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:13:49.718 INFO:teuthology.orchestra.run.smithi195.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:13:49.719 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-26T05:13:49.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T05:13:49.749 INFO:teuthology.orchestra.run.smithi152.stdout:1714107210.8588424 mon.a (mon.0) 503 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T05:13:49.750 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-26T05:13:49.750 INFO:tasks.cephadm:Compressing logs... 2024-04-26T05:13:49.750 DEBUG:teuthology.orchestra.run.smithi152:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:13:49.792 DEBUG:teuthology.orchestra.run.smithi195:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:13:49.815 INFO:teuthology.orchestra.run.smithi195.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:13:49.816 INFO:teuthology.orchestra.run.smithi195.stderr:: No such file or directory 2024-04-26T05:13:49.816 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:13:49.816 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:13:49.816 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-26T05:13:49.816 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-26T05:13:49.825 INFO:teuthology.orchestra.run.smithi152.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:13:49.825 INFO:teuthology.orchestra.run.smithi152.stderr:: No such file or directory 2024-04-26T05:13:49.825 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:13:49.825 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-26T05:13:49.825 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.0.log: 92.7% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-26T05:13:49.826 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.c.log: 93.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:13:49.827 INFO:teuthology.orchestra.run.smithi152.stderr: 93.6% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:13:49.827 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-26T05:13:49.827 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.7.log: /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi195.bqeurw.log 2024-04-26T05:13:49.827 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-26T05:13:49.827 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi195.bqeurw.log: gzip -5 --verbose 62.6% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi195.bqeurw.log.gz 2024-04-26T05:13:49.827 INFO:teuthology.orchestra.run.smithi195.stderr: -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-26T05:13:49.827 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.cephadm.log 87.4% 2024-04-26T05:13:49.828 INFO:teuthology.orchestra.run.smithi195.stderr: -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:13:49.828 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.6.log: /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:13:49.828 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.cephadm.log: 83.6% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:13:49.828 INFO:teuthology.orchestra.run.smithi195.stderr: 89.6% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:13:49.828 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-26T05:13:49.828 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-volume.log: 91.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:13:49.835 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T05:13:49.835 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi152.rnhpxy.log 2024-04-26T05:13:49.835 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.b.log: 91.4% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:13:49.836 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.cephadm.log: 90.1% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:13:49.836 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:13:49.837 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi152.rnhpxy.log: 81.7% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi152.rnhpxy.log.gz 2024-04-26T05:13:49.858 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-volume.log: 91.1% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:13:49.866 INFO:teuthology.orchestra.run.smithi195.stderr: 89.7% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-26T05:13:50.142 INFO:teuthology.orchestra.run.smithi195.stderr: 92.4% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-26T05:13:50.173 INFO:teuthology.orchestra.run.smithi152.stderr: 88.7% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-26T05:13:50.274 INFO:teuthology.orchestra.run.smithi195.stderr: 93.0% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-26T05:13:50.274 INFO:teuthology.orchestra.run.smithi152.stderr: 92.0% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-26T05:13:50.413 INFO:teuthology.orchestra.run.smithi195.stderr: 93.3% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-26T05:13:50.445 INFO:teuthology.orchestra.run.smithi195.stderr: 93.2% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-26T05:13:50.458 INFO:teuthology.orchestra.run.smithi195.stderr: 93.3% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-26T05:13:50.460 INFO:teuthology.orchestra.run.smithi195.stderr: 2024-04-26T05:13:50.460 INFO:teuthology.orchestra.run.smithi195.stderr:real 0m0.655s 2024-04-26T05:13:50.461 INFO:teuthology.orchestra.run.smithi195.stderr:user 0m2.628s 2024-04-26T05:13:50.461 INFO:teuthology.orchestra.run.smithi195.stderr:sys 0m0.113s 2024-04-26T05:13:50.493 INFO:teuthology.orchestra.run.smithi152.stderr: 93.5% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-26T05:13:50.493 INFO:teuthology.orchestra.run.smithi152.stderr: 93.2% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-26T05:13:50.521 INFO:teuthology.orchestra.run.smithi152.stderr: 93.1% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-26T05:13:50.654 INFO:teuthology.orchestra.run.smithi152.stderr: 90.6% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-26T05:13:50.716 INFO:teuthology.orchestra.run.smithi152.stderr: 93.5% -- replaced with /var/log/ceph/164326a4-0388-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-26T05:13:50.719 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-26T05:13:50.719 INFO:teuthology.orchestra.run.smithi152.stderr:real 0m0.913s 2024-04-26T05:13:50.719 INFO:teuthology.orchestra.run.smithi152.stderr:user 0m4.472s 2024-04-26T05:13:50.719 INFO:teuthology.orchestra.run.smithi152.stderr:sys 0m0.167s 2024-04-26T05:13:50.719 INFO:tasks.cephadm:Archiving logs... 2024-04-26T05:13:50.720 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595/remote/smithi152/log 2024-04-26T05:13:50.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:13:51.384 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595/remote/smithi195/log 2024-04-26T05:13:51.385 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:13:51.829 INFO:tasks.cephadm:Removing cluster... 2024-04-26T05:13:51.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 164326a4-0388-11ef-bc93-c7b262605968 --force 2024-04-26T05:13:52.343 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 164326a4-0388-11ef-bc93-c7b262605968 --force 2024-04-26T05:13:52.816 INFO:tasks.cephadm:Removing cephadm ... 2024-04-26T05:13:52.816 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:13:52.832 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:13:52.847 INFO:tasks.cephadm:Teardown complete 2024-04-26T05:13:52.848 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-26T05:13:52.858 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-26T05:13:52.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:13:52.875 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:13:52.963 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-26T05:13:52.972 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-26T05:13:52.973 DEBUG:teuthology.orchestra.run.smithi152:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:13:52.975 DEBUG:teuthology.orchestra.run.smithi195:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:13:52.987 INFO:teuthology.orchestra.run.smithi152.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:13:52.988 INFO:teuthology.orchestra.run.smithi195.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:13:52.992 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:13:52.992 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2024-04-26T05:13:52.992 INFO:teuthology.orchestra.run.smithi152.stdout:^* hv01.front.sepia.ceph.com 3 7 377 93 +970us[ +955us] +/- 56ms 2024-04-26T05:13:52.993 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 30 -1354us[-1354us] +/- 76ms 2024-04-26T05:13:52.993 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 28 -71us[ -71us] +/- 71ms 2024-04-26T05:13:52.993 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:13:52.994 INFO:teuthology.orchestra.run.smithi195.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:13:52.994 INFO:teuthology.orchestra.run.smithi195.stdout:=============================================================================== 2024-04-26T05:13:52.994 INFO:teuthology.orchestra.run.smithi195.stdout:^* hv01.front.sepia.ceph.com 3 6 377 27 +983us[ +979us] +/- 56ms 2024-04-26T05:13:52.994 INFO:teuthology.orchestra.run.smithi195.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 30 -1283us[-1287us] +/- 76ms 2024-04-26T05:13:52.994 INFO:teuthology.orchestra.run.smithi195.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 25 +4930ns[+4930ns] +/- 71ms 2024-04-26T05:13:52.994 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:13:52.995 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-26T05:13:53.004 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-26T05:13:53.005 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-26T05:13:53.094 DEBUG:teuthology.orchestra.run.smithi152:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:13:53.185 DEBUG:teuthology.orchestra.run.smithi195:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:13:53.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:13:53.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:13:53.286 DEBUG:teuthology.orchestra.run.smithi195:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:13:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:13:53.312 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-26T05:13:53.325 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-26T05:13:53.335 INFO:teuthology.task.internal:Duration was 1927.662157 seconds 2024-04-26T05:13:53.336 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-26T05:13:53.347 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-26T05:13:53.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:13:53.350 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:13:53.383 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:13:53.388 INFO:teuthology.orchestra.run.smithi195.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:13:53.666 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-26T05:13:53.667 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi152.front.sepia.ceph.com 2024-04-26T05:13:53.667 DEBUG:teuthology.orchestra.run.smithi152:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:13:53.727 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi195.front.sepia.ceph.com 2024-04-26T05:13:53.728 DEBUG:teuthology.orchestra.run.smithi195:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:13:53.749 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-26T05:13:53.750 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:13:53.772 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:13:53.819 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-26T05:13:53.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:13:53.836 DEBUG:teuthology.orchestra.run.smithi195:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:13:54.264 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-26T05:13:54.275 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-26T05:13:54.275 DEBUG:teuthology.orchestra.run.smithi152:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:13:54.298 DEBUG:teuthology.orchestra.run.smithi195:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:13:54.321 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-26T05:13:54.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:13:54.341 DEBUG:teuthology.orchestra.run.smithi195:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:13:54.360 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = core 2024-04-26T05:13:54.383 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = core 2024-04-26T05:13:54.411 DEBUG:teuthology.orchestra.run.smithi152:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:13:54.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:13:54.431 DEBUG:teuthology.orchestra.run.smithi195:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:13:54.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:13:54.467 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-26T05:13:54.478 INFO:teuthology.task.internal:Transferring archived files... 2024-04-26T05:13:54.478 DEBUG:teuthology.misc:Transferring archived files from smithi152:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595/remote/smithi152 2024-04-26T05:13:54.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:13:54.523 DEBUG:teuthology.misc:Transferring archived files from smithi195:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595/remote/smithi195 2024-04-26T05:13:54.524 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:13:54.560 INFO:teuthology.task.internal:Removing archive directory... 2024-04-26T05:13:54.560 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:13:54.563 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:13:54.608 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-26T05:13:54.621 INFO:teuthology.task.internal:Not uploading archives. 2024-04-26T05:13:54.621 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-26T05:13:54.630 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-26T05:13:54.633 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:13:54.636 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:13:54.648 INFO:teuthology.orchestra.run.smithi152.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:13 /home/ubuntu/cephtest 2024-04-26T05:13:54.663 INFO:teuthology.orchestra.run.smithi195.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:13 /home/ubuntu/cephtest 2024-04-26T05:13:54.664 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-26T05:13:54.675 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-26T05:13:54.700 INFO:teuthology.nuke:Checking targets against current locks 2024-04-26T05:13:54.728 DEBUG:teuthology.nuke:shortname: smithi152 2024-04-26T05:13:54.729 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:13:54.752 DEBUG:teuthology.nuke:shortname: smithi195 2024-04-26T05:13:54.753 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:13:54.756 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:36:02.908114', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOf23yzaLcFuBatbPqDCU5czzKwmjuLZHWW0K08Qqu5BTKYNWPN2wWKcCS7EVgkqEjARquo8CANWGgOYSOayGq4='} 2024-04-26T05:13:54.781 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi195.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674595', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:36:02.910888', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAxaCkGt5M7WGAinUpMquxJmF3Z1AuEWKJlhtoLD5RyKvoNAlXMA4YEb/WHnr7pCmxlqgeE/DXuOwEVv/Geec1I='} 2024-04-26T05:13:54.808 INFO:teuthology.orchestra.console.smithi152:Power off 2024-04-26T05:13:54.809 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:13:54.834 DEBUG:teuthology.orchestra.console.smithi152:power off output: Chassis Power Control: Down/Off 2024-04-26T05:13:54.834 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:13:54.853 INFO:teuthology.orchestra.console.smithi195:Power off 2024-04-26T05:13:54.854 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:13:54.865 DEBUG:teuthology.orchestra.console.smithi152:check power output: Chassis Power is on 2024-04-26T05:13:54.880 DEBUG:teuthology.orchestra.console.smithi195:power off output: Chassis Power Control: Down/Off 2024-04-26T05:13:54.880 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:13:54.905 DEBUG:teuthology.orchestra.console.smithi195:check power output: Chassis Power is on 2024-04-26T05:13:58.866 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:13:58.905 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:13:58.980 DEBUG:teuthology.orchestra.console.smithi152:check power output: Chassis Power is on 2024-04-26T05:13:59.016 DEBUG:teuthology.orchestra.console.smithi195:check power output: Chassis Power is on 2024-04-26T05:14:02.982 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:14:03.017 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:14:03.092 DEBUG:teuthology.orchestra.console.smithi152:check power output: Chassis Power is off 2024-04-26T05:14:03.128 DEBUG:teuthology.orchestra.console.smithi195:check power output: Chassis Power is off 2024-04-26T05:14:03.194 INFO:teuthology.orchestra.console.smithi152:Power off completed 2024-04-26T05:14:03.230 INFO:teuthology.orchestra.console.smithi195:Power off completed 2024-04-26T05:14:03.334 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 1927.6621570587158 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=1ab6213493e94b6fbbfbc49484b5f6d5 status: fail success: false 2024-04-26T05:14:03.334 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T05:14:03.413 INFO:teuthology.run:FAIL